[ { "path": ".coveralls.yml", "content": "service_name: travis-ci" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n\n# vscode\n.vscode/" }, { "path": ".lgtm.yml", "content": "extraction:\n python:\n python_setup:\n version: 3\n index:\n exclude:\n - .git\n #after_prepare:\n # - python3 -m pip install --upgrade --user flake8\n #before_index:\n # - python3 -m flake8 --version # flake8 3.6.0 on CPython 3.6.5 on Linux\n # # stop the build if there are Python syntax errors or undefined names\n # - python3 -m flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics\n # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide\n # - python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics\n" }, { "path": ".travis.yml", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlanguage: python\npython: 3.6\n\n# Safelist: focus Travis' attention on the master and develop branches only.\nbranches:\n only:\n - master\n - develop\n\n#install:\n# - pip3 install -r requirements.txt\n\nbefore_install:\n - sudo apt-get update\n # Install conda.\n - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;\n - bash miniconda.sh -b -p $HOME/miniconda\n - export PATH=\"$HOME/miniconda/bin:$PATH\"\n # Set conda to always \"--yes\" mode.\n - conda config --set always_yes yes --set changeps1 no\n - conda update -q conda\n # Create env and install pytorch\n - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytorch scipy -c pytorch\n - source activate test-environment\n # Coveralls\n - pip install coveralls\n\ninstall:\n - python setup.py install\n\nscript:\n # Test plake8 compatibility.\n #- python3 -m flake8 --version # flake8 3.6.0 on CPython 3.6.5 on Linux\n # stop the build if there are Python syntax errors or undefined names\n #- python3 -m flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics\n # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide\n #- python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics\n # Run unittests in ptp/test.\n #- python -m unittest ptp\n - coverage run -m unittest #discover -s ptp\n # Build documentation.\n # TODO.\n\nafter_success:\n # Coverals.\n - coveralls" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# PyTorchPipe\r\n\r\n![Language](https://img.shields.io/badge/language-Python-blue.svg)\r\n[![GitHub license](https://img.shields.io/github/license/IBM/pytorchpipe.svg)](https://github.com/IBM/pytorchpipe/blob/develop/LICENSE)\r\n[![GitHub version](https://badge.fury.io/gh/IBM%2Fpytorchpipe.svg)](https://badge.fury.io/gh/IBM%2Fpytorchpipe)\r\n\r\n[![Build Status](https://travis-ci.com/IBM/pytorchpipe.svg?branch=develop)](https://travis-ci.com/IBM/pytorchpipe)\r\n[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/IBM/pytorchpipe.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/IBM/pytorchpipe/context:python)\r\n[![Total alerts](https://img.shields.io/lgtm/alerts/g/IBM/pytorchpipe.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/IBM/pytorchpipe/alerts/)\r\n[![Coverage Status](https://coveralls.io/repos/github/IBM/pytorchpipe/badge.svg?branch=develop)](https://coveralls.io/github/IBM/pytorchpipe?branch=develop)\r\n[![Maintainability](https://api.codeclimate.com/v1/badges/e8d37123b856ee5bb10b/maintainability)](https://codeclimate.com/github/IBM/pytorchpipe/maintainability)\r\n\r\n## Description\r\n\r\nPyTorchPipe (PTP) is a component-oriented framework that facilitates development of computational _multi-modal pipelines_ and comparison of diverse neural network-based models.\r\n\r\nPTP frames training and testing procedures as _pipelines_ consisting of many components communicating through data streams.\r\nEach such a stream can consist of several components, including one task instance (providing batches of data), any number of trainable components (models) and additional components providing required transformations and computations.\r\n\r\n\r\n![Alt text](docs/source/img/data_flow_vqa_5_attention_gpu_loaders.png?raw=true \"Exemplary multi-modal data flow diagram\")\r\n\r\n\r\nAs a result, the training & testing procedures are no longer pinned to a specific task or model, and built-in mechanisms for compatibility checking (handshaking), configuration and global variables management & statistics collection facilitate rapid development of complex pipelines and running diverse experiments.\r\n\r\nIn its core, to _accelerate the computations_ on their own, PTP relies on PyTorch and extensively uses its mechanisms for distribution of computations on CPUs/GPUs, including multi-process data loaders and multi-GPU data parallelism.\r\nThe models are _agnostic_ to those operations and one indicates whether to use them in configuration files (data loaders) or by passing adequate argument (--gpu) at run-time.\r\n\r\nPlease refer to the [tutorial presentation](https://zenodo.org/record/3269928) for more details.\r\n\r\n**Datasets:**\r\nPTP focuses on multi-modal reasoning combining vision and language. Currently it offers the following _Tasks_ from the following task, categorized into three domains:\r\n\r\n![Alt text](docs/source/img/components/ptp_tasks.png?raw=true)\r\n\r\nAside of providing batches of samples, the Task class will automatically download the files associated with a given dataset (as long as the dataset is publicly available).\r\nThe diversity of those tasks (and the associated models) proves the flexibility of the framework.\r\nWe are constantly working on incorporation of new Tasks into PTP.\r\n\r\n**Pipelines:**\r\nWhat people typically define as a _model_ in PTP is framed as a _pipeline_, consisting of many inter-connected components, with one or more _Models_ containing trainable elements.\r\nThose components are loosely coupled and care only about the _input streams_ they retrieve and _output streams_ they produce.\r\nThe framework offers full flexibility and it is up to the programmer to choose the _granularity_ of his/her components/models/pipelines.\r\nSuch a decomposition enables one to easily combine many components and models into pipelines, whereas the framework supports loading of pretrained models, freezing during training, saving them to checkpoints etc.\r\n\r\n**Model/Component Zoo:**\r\nPTP provides several ready to use, out of the box models and other, non-trainable (but parametrizable) components.\r\n\r\n\r\n![Alt text](docs/source/img/components/ptp_models.png?raw=true)\r\n\r\nThe model zoo includes several general usage components, such as:\r\n * Feed Forward Network (variable number of Fully Connected layers with activation functions and dropout)\r\n * Recurrent Neural Network (different cell types with activation functions and dropout, a single model can work both as encoder or decoder)\r\n\r\nIt also inludes few models specific for a given domain, but still quite general:\r\n * Convnet Encoder (CNNs with ReLU and MaxPooling, can work with different sizes of images)\r\n * General Image Encoder (wrapping several models from Torch Vision)\r\n * Sentence Embeddings (encoding words using the embedding layer)\r\n\r\nThere are also some classical baselines both for vision like LeNet-5 or language domains, e.g. Seq2Seq (Sequence to Sequence model) or Attention Decoder (RNN-based decoder implementing Bahdanau-style attention).\r\nPTP also offers the several models useful for multi-modal fusion and reasoning.\r\n\r\n![Alt text](docs/source/img/components/ptp_components_others.png?raw=true)\r\n\r\nThe framework also offers components useful when working with language, vision or other types of streams (e.g. tensor transformations).\r\nThere are also several general-purpose components, from components calculating losses and statistics to publishers and viewers.\r\n\r\n**Workers:**\r\nPTP workers are python scripts that are _agnostic_ to the tasks/models/pipelines that they are supposed to work with.\r\nCurrently framework offers three workers:\r\n\r\n * ptp-offline-trainer (a trainer relying on classical methodology interlacing training and validation at the end of every epoch, creates separate instances of training and validation tasks and trains the models by feeding the created pipeline with batches of data, relying on the notion of an _epoch_)\r\n\r\n * ptp-online-trainer (a flexible trainer creating separate instances of training and validation tasks and training the models by feeding the created pipeline with batches of data, relying on the notion of an _episode_)\r\n\r\n * ptp-processor (performing one pass over the all samples returned by a given task instance, useful for collecting scores on test set, answers for submissions to competitions etc.)\r\n\r\n\r\n## Installation\r\n\r\nPTP relies on [PyTorch](https://github.com/pytorch/pytorch), so you need to install it first.\r\nPlease refer to the official installation [guide](https://github.com/pytorch/pytorch#installation) for details.\r\nIt is easily installable via conda_, or you can compile it from source to optimize it for your machine.\r\n\r\nPTP is not (yet) available as a [pip](https://pip.pypa.io/en/stable/quickstart/) package, or on [conda](https://anaconda.org/pytorch/pytorch).\r\nHowever, we provide the `setup.py` script and recommend to use it for installation.\r\nFirst please clone the project repository:\r\n\r\n```console\r\ngit clone git@github.com:IBM/pytorchpipe.git\r\ncd pytorchpipe/\r\n```\r\n\r\nNext, install the dependencies by running:\r\n\r\n```console\r\n python setup.py develop\r\n```\r\n\r\nThis command will install all dependencies via pip_, while still enabling you to change the code of the existing components/workers and running them by calling the associated ``ptp-*`` commands.\r\nMore in that subject can be found in the following blog post on [dev_mode](https://setuptools.readthedocs.io/en/latest/setuptools.html#development-mode).\r\n\r\n\r\n## Quick start: MNIST image classification with a simple ConvNet model\r\n\r\nPlease consider a simple ConvNet model consisting of two parts: \r\n * few convolutional layers accepting the MNIST images and returning feature maps being, in general, a 4D tensor (first dimension being the batch size, a rule of thumb in PTP),\r\n * one (or more) dense layers that accept the (flattened) feature maps and return predictions in the form of logarithm of probability distributions (LogSoftmax as last non-linearity).\r\n\r\n### Training the model\r\n\r\nAssume that we will use ```NLL Loss``` function, and, besides, want to monitor the ```Accuracy``` statistics.\r\nThe resulting pipeline is presented below.\r\nThe additional ```Answer Decoder``` component translates the predictions into class names, whereas ```Stream Viewer``` displays content of the indicated data streams for a single sample randomly picked from the batch.\r\n\r\n\r\n![Alt text](docs/source/img/1_tutorials/data_flow_tutorial_mnist_1_training.png?raw=true \"Trainining of a simple ConvNet model on MNIST dataset\")\r\n\r\n__Note__: The associated ```mnist_classification_convnet_softmax.yml``` configuration file can be found in ```configs/tutorials``` folder.\r\n\r\nWe will train the model with _ptp-offline-trainer_, a general _worker_ script that follows the classical training-validation, epoch-based methodology.\r\nThis means, that despite the presence of three sections (associated with training, validation and test splits of the MNIST dataset) the trainer will consider only the content of ``training`` and ```validation``` sections (plus ```pipeline```, containing the definition of the whole pipeline).\r\nLet's run the training by calling the following from the command line:\r\n\r\n```console\r\nptp-offline-trainer --c configs/tutorials/mnist_classification_convnet_softmax.yml\r\n```\r\n\r\n__Note__: Please call ```offline-trainer --h``` to learn more about the run-time arguments. In order to understand the structure of the main configuration file please look at the default configuration file of the trainer located in ```configs/default/workers``` folder.\r\n\r\nThe trainer will log on the console training and validation statistis, along with additional information logged by the components, e.g. contents of the streams:\r\n\r\n```console\r\n[2019-07-05 13:31:44] - INFO - OfflineTrainer >>> episode 006000; epoch 06; loss 0.1968410313; accuracy 0.9219\r\n[2019-07-05 13:31:45] - INFO - OfflineTrainer >>> End of epoch: 6\r\n================================================================================\r\n[2019-07-05 13:31:45] - INFO - OfflineTrainer >>> episode 006019; episodes_aggregated 000860; epoch 06; loss 0.1799264401; loss_min 0.0302138925; loss_max 0.5467863679; loss_std 0.0761705562; accuracy 0.94593; accuracy_std 0.02871 [Full Training]\r\n[2019-07-05 13:31:45] - INFO - OfflineTrainer >>> Validating over the entire validation set (5000 samples in 79 episodes)\r\n[2019-07-05 13:31:45] - INFO - stream_viewer >>> Showing selected streams for sample 20 (index: 55358):\r\n 'labels': One\r\n 'targets': 1\r\n 'predictions': tensor([-1.1452e+01, -1.6804e-03, -1.1357e+01, -1.1923e+01, -6.6160e+00,\r\n -1.4658e+01, -9.6191e+00, -8.6472e+00, -9.6082e+00, -1.3505e+01])\r\n 'predicted_answers': One\r\n```\r\n\r\nPlease note that whenever the validation loss goes down, the trainer automatically will save the pipeline to the checkpoint file:\r\n\r\n```console\r\n[2019-07-05 13:31:47] - INFO - OfflineTrainer >>> episode 006019; episodes_aggregated 000079; epoch 06; loss 0.1563445479; loss_min 0.0299939774; loss_max 0.5055227876; loss_std 0.0854654983; accuracy 0.95740; accuracy_std 0.02495 [Full Validation]\r\n[2019-07-05 13:31:47] - INFO - mnist_classification_convnet_softmax >>> Exporting pipeline 'mnist_classification_convnet_softmax' parameters to checkpoint:\r\n /users/tomaszkornuta/experiments/mnist/mnist_classification_convnet_softmax/20190705_132624/checkpoints/mnist_classification_convnet_softmax_best.pt\r\n + Model 'image_encoder' [ConvNetEncoder] params saved\r\n + Model 'classifier' [FeedForwardNetwork] params saved\r\n```\r\n\r\nAfter the training finsh the trainer will inform about the termination reason and indicate where the experiment files (model checkpoint, log files, statistics etc.) can be found:\r\n\r\n```console\r\n[2019-07-05 13:32:33] - INFO - mnist_classification_convnet_softmax >>> Updated training status in checkpoint:\r\n /users/tomaszkornuta/experiments/mnist/mnist_classification_convnet_softmax/20190705_132624/checkpoints/mnist_classification_convnet_softmax_best.pt\r\n[2019-07-05 13:32:33] - INFO - OfflineTrainer >>>\r\n================================================================================\r\n[2019-07-05 13:32:33] - INFO - OfflineTrainer >>> Training finished because Converged (Full Validation Loss went below Loss Stop threshold of 0.15)\r\n[2019-07-05 13:32:33] - INFO - OfflineTrainer >>> Experiment finished!\r\n[2019-07-05 13:32:33] - INFO - OfflineTrainer >>> Experiment logged to: /users/tomaszkornuta/experiments/mnist/mnist_classification_convnet_softmax/20190705_132624/\r\n```\r\n\r\n\r\n### Testing the model\r\n\r\nIn order to test the model generalization we will use _ptp-processor_, yet another general _worker_ script that performs a single pass over the indicated set.\r\n\r\n\r\n![Alt text](docs/source/img/1_tutorials/data_flow_tutorial_mnist_2_test.png?raw=true \"Test of the pretrained model on test split of the MNIST dataset \")\r\n\r\n\r\n```console\r\nptp-processor --load /users/tomaszkornuta/experiments/mnist/mnist_classification_convnet_softmax/20190705_132624/checkpoints/mnist_classification_convnet_softmax_best.pt\r\n```\r\n\r\n__Note__: _ptp-processor_ uses the content of _test_ section as default, but it can be changed at run-time. Please call ```ptp-processor --h``` to learn about the available run-time arguments.\r\n\r\n\r\n```console\r\n[2019-07-05 13:34:41] - INFO - Processor >>> episode 000313; episodes_aggregated 000157; loss 0.1464060694; loss_min 0.0352710858; loss_max 0.3801054060; loss_std 0.0669835582; accuracy 0.95770; accuracy_std 0.02471 [Full Set]\r\n[2019-07-05 13:34:41] - INFO - Processor >>> Experiment logged to: /users/tomaszkornuta/experiments/mnist/mnist_classification_convnet_softmax/20190705_132624/test_20190705_133436/\r\n```\r\n\r\n__Note__: Please analyze the ```mnist_classification_convnet_softmax.yml``` configuration file (located in ```configs/tutorials``` directory). Keep in mind that:\r\n * all components come with default configuration files, located in ```configs/default/components``` folders,\r\n * all workers come with default configuration files, located in ```configs/default/workers``` folders.\r\n## Documentation\r\n\r\nCurrently PTP does not have an on-line documentation.\r\nHowever, there are high-quality comments in all source/configuration files, that will be used for automatic generation of documentation (Sphinx + ReadTheDocs).\r\nBesides, we have shared a [tutorial presentation](https://zenodo.org/record/3269928) explaining motivations and core concepts as well as providing hints how to use the tool and develop your own solutions.\r\n\r\n\r\n## Contributions\r\n\r\nPTP is open for external contributions.\r\nWe follow the [Git Branching Model](https://nvie.com/posts/a-successful-git-branching-model/), in short:\r\n * ```develop``` branch is the main branch, ```master``` branch is for used for releases only\r\n * all changes are integrated by merging pull requests from feat/fix/other branches\r\n * PTP is integrated with several DevOps monitoring the quality of code/pull requests\r\n * we strongly encourage unit testing and Test-Driven Development\r\n * we use projects and kanban to monitor issues/progress/etc.\r\n\r\n\r\n## Maintainers\r\n\r\nA project of the Machine Intelligence team, IBM Research AI, Almaden Research Center.\r\n\r\n* Tomasz Kornuta (tkornut@us.ibm.com)\r\n\r\n[![HitCount](http://hits.dwyl.io/tkornut/tkornut/pytorchpipe.svg)](http://hits.dwyl.io/tkornut/tkornut/pytorchpipe)\r\n" }, { "path": "configs/cifar100/cifar100_classification_convnet_softmax.yml", "content": "# Load config defining CIFAR100 tasks for training, validation and testing.\ndefault_configs: cifar100/default_cifar100.yml\n\n# Definition of the pipeline.\npipeline:\n\n # Model consisting of two components.\n image_encoder:\n priority: 1.1\n type: ConvNetEncoder\n\n # Reshape inputs\n reshaper:\n priority: 1.2\n type: ReshapeTensor\n input_dims: [-1, 16, 2, 2]\n output_dims: [-1, 64]\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Image classifier.\n classifier:\n priority: 1.3\n type: FeedForwardNetwork \n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_fine_classes\n\n#: pipeline\n" }, { "path": "configs/cifar100/default_cifar100.yml", "content": "# Training parameters:\ntraining:\n task: \n type: CIFAR100\n batch_size: &b 1024\n use_train_data: True\n # Use sampler that operates on a subset.\n dataloader:\n num_workers: 10\n # shuffle: False\n sampler:\n type: SubsetRandomSampler\n indices: [0, 45000]\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.05\n early_stop_validations: -1\n episode_limit: 10000\n epoch_limit: 10\n\n# Validation parameters:\nvalidation:\n #partial_validation_interval: 100\n task:\n type: CIFAR100\n batch_size: *b\n use_train_data: True # True because we are splitting the training set to: validation and training\n #resize: [32, 32]\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [45000, 50000]\n\n# Testing parameters:\ntest:\n task:\n type: MNIST\n batch_size: *b\n use_train_data: False\n #resize: [32, 32]\n\npipeline:\n disable: image_viewer\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.0\n streams:\n targets: fine_targets\n\n # Statistics.\n batch_size:\n priority: 100.0\n type: BatchSizeStatistics\n streams:\n targets: fine_targets\n\n accuracy:\n priority: 100.1\n type: AccuracyStatistics\n streams:\n targets: fine_targets\n\n\n precision_recall:\n priority: 100.2\n type: PrecisionRecallStatistics\n use_word_mappings: True\n #show_class_scores: True\n globals:\n word_mappings: fine_label_word_mappings\n streams:\n targets: fine_targets\n\n answer_decoder:\n priority: 100.3\n type: WordDecoder\n import_word_mappings_from_globals: True\n globals:\n word_mappings: fine_label_word_mappings\n streams:\n inputs: predictions\n outputs: answers\n\n stream_viewer:\n priority: 100.4\n type: StreamViewer\n input_streams: coarse_targets, coarse_labels, fine_targets, fine_labels, answers\n\n image_viewer:\n priority: 100.5\n type: ImageViewer\n streams:\n images: inputs\n labels: fine_labels\n answers: coarse_labels \n\n" }, { "path": "configs/clevr/clevr_all_vgg_glove_lstm_concat_ffn.yml", "content": "# Load config defining CLEVR tasks for training, validation and testing.\ndefault_configs: clevr/default_clevr.yml\n\n# Resize and normalize images - in all sets.\ntraining:\n task: \n resize_image: [224, 224]\n image_preprocessing: normalize\n\nvalidation:\n task: \n resize_image: [224, 224]\n image_preprocessing: normalize\n\ntest:\n task: \n resize_image: [224, 224]\n image_preprocessing: normalize\n\n# Definition of the pipeline.\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n keys: [question_encoder_output_size, image_encoder_output_size]\n values: [100, 100]\n\n ##################################################################\n # 1st pipeline: question.\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n # Lowercase all letters + remove punctuation (reduced vocabulary of 80 words instead of 87)\n preprocessing: all\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/CLEVR_v1.0\n word_mappings_file: questions.all.word.mappings.lowercase.csv\n export_word_mappings_to_globals: True\n globals:\n word_mappings: question_word_mappings\n vocabulary_size: num_question_words\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Zero\n hidden_size: 50\n # Turn of softmax.\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ##################################################################\n # 2nd subpipeline: image.\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model_type: vgg16\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ##################################################################\n # 3rd subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 3.1\n input_streams: [question_activations,image_activations]\n dim: 1 # default\n input_dims: [[-1,100],[-1,100]]\n output_dims: [-1,200]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_size\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 3.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatenated_size\n prediction_size: num_answers\n\n#: pipeline\n" }, { "path": "configs/clevr/clevr_image_convnet_ffn.yml", "content": "# Load config defining CLEVR tasks for training, validation and testing.\ndefault_configs: clevr/default_clevr.yml\n\n# Definition of the pipeline.\npipeline:\n\n # Model consisting of two components.\n image_encoder:\n priority: 1.1\n type: ConvNetEncoder\n streams:\n inputs: images\n\n # Reshape inputs\n reshaper:\n priority: 1.2\n type: ReshapeTensor\n input_dims: [-1, 16, 58, 38]\n output_dims: [-1, 35264]\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Image classifier.\n classifier:\n priority: 1.3\n type: FeedForwardNetwork \n hidden_sizes: [1000]\n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_answers\n\n#: pipeline\n" }, { "path": "configs/clevr/clevr_question_glove_lstm.yml", "content": "# Load config defining CLEVR tasks for training, validation and testing.\ndefault_configs: clevr/default_clevr.yml\n\n# This is unimodal (questino-based) baseline, thus stop streaming images - in all sets.\ntraining:\n task: \n stream_images: False\n\nvalidation:\n task: \n stream_images: False\n\ntest:\n task: \n stream_images: False\n\n# Definition of the pipeline.\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n # Lowercase all letters + remove punctuation (reduced vocabulary of 80 words instead of 87)\n preprocessing: all\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/CLEVR_v1.0\n word_mappings_file: questions.all.word.mappings.lowercase.csv\n export_word_mappings_to_globals: True\n globals:\n word_mappings: question_word_mappings\n vocabulary_size: num_question_words\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Zero\n hidden_size: 50\n streams:\n inputs: embedded_questions\n globals:\n input_size: embeddings_size\n prediction_size: num_answers\n\n\n#: pipeline\n" }, { "path": "configs/clevr/default_clevr.yml", "content": "# Training parameters:\ntraining:\n task: \n type: CLEVR\n batch_size: &b 64\n split: training\n #resize_image: [224, 224]\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.05\n early_stop_validations: -1\n episode_limit: 10000\n epoch_limit: 10\n\n# Validation parameters:\nvalidation:\n task:\n type: CLEVR\n batch_size: *b\n split: validation\n #resize_image: [224, 224]\n\n# Testing parameters:\ntest:\n task:\n type: CLEVR\n batch_size: *b\n split: test\n #resize_image: [224, 224]\n\npipeline:\n name: tmp\n disable: image_viewer\n\n label_to_target:\n type: LabelIndexer\n priority: 0.1\n # Load word mappings for answers.\n data_folder: ~/data/CLEVR_v1.0\n word_mappings_file: answers.all.word.mappings.csv\n export_word_mappings_to_globals: True\n globals:\n word_mappings: answer_word_mappings\n vocabulary_size: num_answers\n streams:\n inputs: answers \n outputs: target_answers\n\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.1\n streams:\n targets: target_answers\n\n # Statistics.\n batch_size:\n priority: 100.0\n type: BatchSizeStatistics\n\n accuracy:\n priority: 100.1\n type: AccuracyStatistics\n streams:\n targets: target_answers\n\n precision_recall:\n priority: 100.2\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n globals:\n word_mappings: answer_word_mappings\n streams:\n targets: target_answers\n\n answer_decoder:\n priority: 100.3\n type: WordDecoder\n import_word_mappings_from_globals: True\n globals:\n word_mappings: answer_word_mappings\n streams:\n inputs: predictions\n outputs: predicted_answers\n\n stream_viewer:\n priority: 100.4\n type: StreamViewer\n input_streams: indices, questions, target_answers, predicted_answers\n\n #image_viewer:\n # priority: 100.5\n # type: ImageViewer\n # streams:\n # images: inputs\n # labels: labels\n # answers: answers\n\n" }, { "path": "configs/default/components/language/bow_encoder.yml", "content": "# This file defines the default values for BOW Encoder.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Length of the bag-of-word vector.\n bow_size: bow_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/language/label_indexer.yml", "content": "# This file defines the default values for LabelIndexer.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/'\n\n# Source files that will be used to create the vocabulary (LOADED)\nsource_vocabulary_files: ''\n\n# Additional tokens that will be added to vocabulary (LOADED)\nadditional_tokens: ''\n\n# File containing word (LOADED)\nword_mappings_file: 'word_mappings.csv'\n\n# HACK: This key is useless here, but needed by parent class. Should be removed/fixed in the future\nexport_pad_index_to_globals: False\n\n# If set, component will always (re)generate the vocabulary (LOADED)\nregenerate: False \n\n# Flag informing whether word mappings will be imported from globals (LOADED)\nimport_word_mappings_from_globals: False\n\n# Flag informing whether word mappings will be exported to globals (LOADED)\nexport_word_mappings_to_globals: False\n\n# Value that will be used when word is out of vocabulary (LOADED)\n# (Mask for that element will be 0 as well)\n# -100 is the default value used by PyTroch loss functions to specify\n# target values that will ignored and does not contribute to the input gradient.\n# (ignore_index=-100)\nout_of_vocabulary_value: -100\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # The loaded/exported word mappings (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n word_mappings: word_mappings\n\n # Size of the vocabulary (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n vocabulary_size: vocabulary_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/language/sentence_indexer.yml", "content": "# This file defines the default values for Sentence Indexer.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/'\n\n# Source files that will be used to create the vocabulary (LOADED)\nsource_vocabulary_files: ''\n\n# Additional tokens that will be added to vocabulary (LOADED)\n# This list can be extended, but and are special tokens.\n# is ALWAYS used for padding shorter sequences.\nadditional_tokens: ','\n\n# Enable (end of sequence) token.\neos_token: False\n\n# HACK: This key is useless here, but needed by parent class. Should be removed/fixed in the future\nexport_pad_index_to_globals: False\n\n# File containing word (LOADED)\nword_mappings_file: 'word_mappings.csv'\n\n# If set, component will always (re)generate the vocabulary (LOADED)\nregenerate: False \n\n# Flag informing whether word mappings will be imported from globals (LOADED)\nimport_word_mappings_from_globals: False\n\n# Flag informing whether word mappings will be exported to globals (LOADED)\nexport_word_mappings_to_globals: False\n\n# Fixed padding length\n# -1 -> For each batch, automatically pad to the length of the longest sequence of the batch\n# (variable from batch to batch)\n# > 0 -> Pad each pad to the chosen length (fixed for all batches)\nfixed_padding: -1\n\n# Operation mode. If 'reverse' is True, then it will change indices into words (LOADED)\nreverse: False\n\n# Flag indicating whether inputs are represented as distributions or indices (LOADED)\n# Options: True (expects distribution for each input item in sequence)\n# False (expects indices (max args))\nuse_input_distributions: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # The loaded/exported word mappings (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n word_mappings: word_mappings\n\n # Size of the vocabulary (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n vocabulary_size: vocabulary_size\n\n # Index of the token\n # Will be set only if `export_pad_mapping_to_globals == True`\n pad_index: pad_index\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/language/sentence_one_hot_encoder.yml", "content": "# This file defines the default values for Sentence 1-hot Encoder.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/'\n\n# Source files that will be used to create the vocabulary (LOADED)\nsource_vocabulary_files: ''\n\n# Additional tokens that will be added to vocabulary (LOADED)\nadditional_tokens: ''\n\n# File containing word (LOADED)\nword_mappings_file: 'word_mappings.csv'\n\n# HACK: This key is useless here, but needed by parent class. Should be removed/fixed in the future\nexport_pad_index_to_globals: False\n\n# If set, component will always (re)generate the vocabulary (LOADED)\nregenerate: False \n\n# Flag informing whether word mappings will be imported from globals (LOADED)\nimport_word_mappings_from_globals: False\n\n# Flag informing whether word mappings will be exported to globals (LOADED)\nexport_word_mappings_to_globals: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # The loaded/exported word mappings (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n word_mappings: word_mappings\n\n # Size of the vocabulary (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n vocabulary_size: vocabulary_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/language/sentence_tokenizer.yml", "content": "# This file defines the default values for Sentence Tokenizer.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Operation mode (LOADED)\n# False: sentence -> list of strings, True: list of strings -> sentence.\ndetokenize: False \n\n# Select applied preprocessing/augmentations (LOADED)\n# Use one (or more) of the transformations:\n# none | lowercase | remove_punctuation | all\n# Accepted formats: a,b,c or [a,b,c]\npreprocessing: none\n\n# List of characters to be removed \nremove_characters: ''\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input (detokenized or tokenized) sentences (INPUT)\n inputs: inputs\n\n # Stream containing output (tokenized or detokenized) sentences (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/language/word_decoder.yml", "content": "# This file defines the default values for Word Decoder.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/'\n\n# Source files that will be used to create the vocabulary (LOADED)\nsource_vocabulary_files: ''\n\n# Additional tokens that will be added to vocabulary (LOADED)\nadditional_tokens: ''\n\n# HACK: This key is useless here, but needed by parent class. Should be removed/fixed in the future\nexport_pad_index_to_globals: False\n\n# File containing word (LOADED)\nword_mappings_file: 'word_mappings.csv'\n\n# If set, component will always (re)generate the vocabulary (LOADED)\nregenerate: False \n\n# Flag informing whether word mappings will be imported from globals (LOADED)\nimport_word_mappings_from_globals: False\n\n# Flag informing whether word mappings will be exported to globals (LOADED)\nexport_word_mappings_to_globals: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output words (OUTPUT)\n outputs: outputs\n\nglobals: \n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # The loaded/exported word mappings (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n word_mappings: word_mappings\n\n # Size of the vocabulary (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n vocabulary_size: vocabulary_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/losses/nll_loss.yml", "content": "# This file defines the default values for the NLL Loss.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Number of dimensions of targets, so the loss can work with \n# diffent inputs/targets (LOADED)\nnum_targets_dims: 1\n\n# Loss function (LOADED)\n# Options: NLLLoss | CrossEntropyLoss (NOT OPERATIONAL YET!)\n# loss_function: NLLLoss\n\n# When set to True, performs masking of selected samples from batch (LOADED)\nuse_masking: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing targets (label ids) (INPUT)\n targets: targets\n\n # Stream containing batch of predictions (INPUT)\n predictions: predictions\n\n # Stream containing masks used for masking of selected samples from batch (INPUT)\n masks: masks\n\n # Stream containing loss (OUTPUT)\n loss: loss\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Target value to ignore (masking)\n ignore_index: ignore_index\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/masking/join_masked_predictions.yml", "content": "# This file defines the default values for the Join Masked Predictions component.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# List of input stream names, each containing batch of predictions (LOADED)\ninput_prediction_streams: ''\n\n# List of input stream names, each containing batch of masks (LOADED)\ninput_mask_streams: ''\n\n# List of word mapping names - those will be loaded from globals (LOADED)\ninput_word_mappings: ''\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of output strings (OUTPUT)\n output_strings: output_strings\n\n # Stream containing batch of output indices (OUTPUT)\n # WARNING: As performed operations are not differentiable,\n # those indices cannot be used for e.g. calculation of loss!!\n output_indices: output_indices\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Vocabulary used to produce output strings (RETRIEVED)\n output_word_mappings: output_word_mappings\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/masking/string_to_mask.yml", "content": "# This file defines the default values for the String To Mask component.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Value that will be used when word is out of vocabulary (LOADED)\n# (Mask for that element will be 0 as well)\n# -100 is the default value used by PyTroch loss functions to specify\n# target values that will ignored and does not contribute to the input gradient.\n# (ignore_index=-100)\nout_of_vocabulary_value: -100\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input strings (INPUT)\n strings: strings\n\n # Stream containing output masks (OUTPUT)\n masks: masks\n\n # Stream containing output indices (OUTPUT)\n string_indices: string_indices\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Vocabulary used to produce masks and indices (RETRIEVED)\n word_mappings: word_mappings\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/general_usage/attention_decoder.yml", "content": "# This file defines the default values for the RNN model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Size of the hidden state (LOADED)\nhidden_size: 100\n\n# Wether to include the last hidden state in the outputs\noutput_last_state: False\n\n# Type of recurrent cell (LOADED)\n# -> Only GRU is supported\n\n# Number of \"stacked\" layers (LOADED)\n# -> Only a single layer is supported\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Prediction mode (LOADED)\n# Options: \n# * Dense (passes every activation through output layer) |\n# * Last (passes only the last activation though output layer) |\n# * None (all outputs are discarded)\nprediction_mode: Dense\n\n# Enable FFN layer at the output of the RNN (before eventual feed back in the case of autoregression).\n# Useful if the raw outputs of the RNN are needed, for attention encoder-decoder for example.\nffn_output: True\n\n# Length of generated output sequence (LOADED)\n# User must set it per task, as it is task specific.\nautoregression_length: 10\n\n# If true, output of the last layer will be additionally processed with Log Softmax (LOADED)\nuse_logsoftmax: True\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoder outputs (INPUT)\n inputs: inputs\n\n # Stream containing the inital state of the RNN (INPUT)\n # The stream will be actually created only if `inital_state: Input`\n input_state: input_state\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\n # Stream containing the final output state of the RNN (output)\n # The stream will be actually created only if `output_last_state: True`\n output_state: output_state\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the input (RETRIEVED)\n input_size: input_size\n\n # Size of the prediction (RETRIEVED)\n prediction_size: prediction_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/general_usage/feed_forward_network.yml", "content": "# This file defines the default values for the Multi-Layer Feed-Forward Network.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Optional (LOADED)\n# Number of hidden layers, along with their sizes (numbers of neurons).\n# hidden_sizes: [dim hidden 1, dim hidden 2, ...]\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# If true, output of the last layer will be additionally processed with Log Softmax (LOADED)\nuse_logsoftmax: True\n\n# Number of dimensions, where:\n# - 2 means [Batch size, Input size]\n# - n means [Batch size, dim 1, ..., dim n-2, Input size]\n# And the FFN is broadcasted over the last (Input Size) Dimension.\n# Also, all the dimensions sizes but the last are conserved, as the FFN is applied over the last dimension.\ndimensions: 2\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of inputs (INPUT)\n inputs: inputs\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the input (RETRIEVED)\n input_size: input_size\n\n # Size of the prediction (RETRIEVED)\n prediction_size: prediction_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/general_usage/recurrent_neural_network.yml", "content": "# This file defines the default values for the RNN model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Size of the hidden state (LOADED)\nhidden_size: 100\n\n# Flag informing the model to learn the intial state (h0/c0) (LOADED)\n# When false, (c0/c0) will be initialized as zeros.\n\n# Input mode (LOADED)\n# Options:\n# * Dense (every iteration expects an input)\n# * Autoregression_First (Autoregression, expects an input for the first iteration)\n# * Autoregression_None (Autoregression, first input will be a null vector)\ninput_mode: Dense\n\n# Prediction mode (LOADED)\n# Options:\n# * Dense (passes every activation through output layer) |\n# * Last (passes only the last activation through output layer) |\n# * None (all outputs are discarded)\nprediction_mode: Dense\n\n# Maximal length of generated output sequence when working in auto-regression mode (LOADED)\n# User must set it per task, as it is task specific.\n# max_autoregression_length: x\n\n# Initial state type (LOADED)\n# * Zero (Vector of zeros, not trainable)\n# * Trainable (xavier initialization, trainable)\n# * Input (the initial hidden state comes from an input stream)\ninitial_state: Trainable\n\n# Type of recurrent cell (LOADED)\n# Options: LSTM | GRU | RNN_TANH | RNN_RELU\ncell_type: LSTM\n\n# Number of \"stacked\" layers (LOADED)\nnum_layers: 1\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Enable FFN layer at the output of the RNN (LOADED)\n# Useful if the raw outputs of the RNN are needed, for attention encoder-decoder for example.\nuse_output_layer: True\n\n# Wether to include the last hidden state in the outputs\noutput_last_state: False\n\n# If true, output of the last layer will be additionally processed with Log Softmax (LOADED)\nuse_logsoftmax: True\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of images (INPUT)\n inputs: inputs\n\n # Stream containing the inital state of the RNN (INPUT)\n # The stream will be actually created only if `inital_state: Input`\n input_state: input_state\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\n # Stream containing the final output state of the RNN (output)\n # The stream will be actually created only if `output_last_state: True`\n output_state: output_state\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the input (RETRIEVED)\n input_size: input_size\n\n # Size of the prediction (RETRIEVED)\n prediction_size: prediction_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/models/general_usage/seq2seq.yml", "content": "# This file defines the default values for the RNN model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Size of the hidden state (LOADED)\nhidden_size: 100\n\n# Flag informing the model to learn the intial state (h0/c0) (LOADED)\n# When false, (c0/c0) will be initialized as zeros.\n\n# Initial state type:\n# * Zero (null vector)\n# * Trainable (xavier initialization, trainable)\n# * Input (the initial hidden state comes from an input stream)\ninitial_state: Trainable\n\n# Wether to include the last hidden state in the outputs\noutput_last_state: False\n\n# Type of recurrent cell (LOADED)\n# Options: LSTM | GRU | RNN_TANH | RNN_RELU\ncell_type: LSTM\n\n# Number of \"stacked\" layers (LOADED)\nnum_layers: 1\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Prediction mode (LOADED)\n# Options: \n# * Dense (passes every activation through output layer) |\n# * Last (passes only the last activation though output layer) |\n# * None (all outputs are discarded)\nprediction_mode: Dense\n\n# Input mode\n# Options:\n# * Dense (every iteration expects an input)\n# * Autoregression_First (Autoregression, expects an input for the first iteration)\n# * Autoregression_None (Autoregression, first input will be a null vector)\ninput_mode: Dense\n\nautoregression_length: 50\n\n# If true, output of the last layer will be additionally processed with Log Softmax (LOADED)\nuse_logsoftmax: True\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of images (INPUT)\n inputs: inputs\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the input (RETRIEVED)\n input_size: input_size\n\n # Size of the prediction (RETRIEVED)\n prediction_size: prediction_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/language/index_embeddings.yml", "content": "# This file defines the default values for the Index Embeddings.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Size of the embeddings (LOADED)\n# Need to be set by the user.\n# embeddings_size: 100\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of inputs (INPUT)\n inputs: inputs\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the vocabulary (RETRIEVED)\n vocab_size: vocab_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the embeddings (SET)\n # It is exported to globals, so other components can use it during\n # their initialization.\n embeddings_size: embeddings_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/language/sentence_embeddings.yml", "content": "# This file defines the default values for the Sentence Embeddings.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/'\n\n# Source files that will be used to create the vocabulary (LOADED)\nsource_vocabulary_files: ''\n\n# Additional tokens that will be added to vocabulary (LOADED)\n# This list can be extended, but and are special tokens.\n# is ALWAYS used for padding shorter sequences.\nadditional_tokens: ''\n\n# Enable (end of sequence) token.\neos_token: False\n\nexport_pad_index_to_globals: False\n\n# File containing word (LOADED)\nword_mappings_file: 'word_mappings.csv'\n\n# If set, component will always (re)generate the vocabulary (LOADED)\nregenerate: False \n\n# Flag informing whether word mappings will be imported from globals (LOADED)\nimport_word_mappings_from_globals: False\n\n# Flag informing whether word mappings will be exported to globals (LOADED)\nexport_word_mappings_to_globals: False\n\n# Fixed padding length\n# -1 -> For each batch, automatically pad to the length of the longest sequence of the batch\n# (variable from batch to batch)\n# > 0 -> Pad each pad to the chosen length (fixed for all batches)\nfixed_padding: -1\n\n# File containing pretrained embeddings (LOADED)\n# Empty means that no embeddings will be loaded.\n# Options: \n# '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt |\n# glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\npretrained_embeddings_file: ''\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of inputs (INPUT)\n inputs: inputs\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # The loaded/exported word mappings (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n word_mappings: word_mappings\n\n # Size of the vocabulary (RETRIEVED/SET)\n # This depends on the import/export configuration flags above.\n vocabulary_size: vocabulary_size\n\n # Size of the embeddings (SET)\n # It is exported to globals, so other components can use it during\n # their initialization.\n embeddings_size: embeddings_size\n\n # Index of the token\n # Will be set only if `export_pad_mapping_to_globals == True`\n pad_index: pad_index\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/compact_bilinear_pooling.yml", "content": "# This file defines the default values for the Multimodal Compact Bilinear Pooling model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Parameter denoting whether projection matrices are trainable (LOADED)\n# Setting flag that to true will result in trainable, dense (i.e. not \"sketch\") projection layers.\ntrainable_projections: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded images (INPUT)\n image_encodings: image_encodings\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the image encodings input (RETRIEVED)\n image_encoding_size: image_encoding_size\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n # Size of the output (RETRIEVED)\n output_size: output_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/factorized_bilinear_pooling.yml", "content": "# This file defines the default values for the FactorizedBilinearPooling model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Size of the latent space (LOADED)\nlatent_size: 100\n\n# Factor used for sum pooling (LOADED)\npool_factor: 2\n\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded images (INPUT)\n image_encodings: image_encodings\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the image encodings input (RETRIEVED)\n image_encoding_size: image_encoding_size\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output (SET)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/low_rank_bilinear_pooling.yml", "content": "# This file defines the default values for the LowRankBilinearPooling model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded images (INPUT)\n image_encodings: image_encodings\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the image encodings input (RETRIEVED)\n image_encoding_size: image_encoding_size\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n # Size of the output (RETRIEVED)\n output_size: output_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/question_driven_attention.yml", "content": "# This file defines the default values for the QuestionDrivenAttention model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Size of the latent space (LOADED)\nlatent_size: 100\n\n# Number of attention heads (LOADED)\nnum_attention_heads: 2\n\n# Type of output returned\n# Options: Image | Fusion\n# Details: attention-weighted image | concatenation of attention-weighted image and RNN encoded question\noutput_mode: Fusion\n\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded images (INPUT)\n feature_maps: feature_maps\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Height of the features tensor (RETRIEVED)\n feature_maps_height: feature_maps_height\n\n # Width of the features tensor (RETRIEVED)\n feature_maps_width: feature_maps_width\n\n # Depth of the features tensor (RETRIEVED)\n feature_maps_depth: feature_maps_depth\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output (SET)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/relational_network.yml", "content": "# This file defines the default values for the LowRankBilinearPooling model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Number of layers along with their sizes (numbers of neurons) of g_theta network (LOADED)\ng_theta_sizes: [256, 256, 256]\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded images (INPUT)\n feature_maps: feature_maps\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Height of the features tensor (RETRIEVED)\n feature_maps_height: feature_maps_height\n\n # Width of the features tensor (RETRIEVED)\n feature_maps_width: feature_maps_width\n\n # Depth of the features tensor (RETRIEVED)\n feature_maps_depth: feature_maps_depth\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output (SET)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/multi_modal_reasoning/self_attention.yml", "content": "# This file defines the default values for the Self_Attention model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dropout rate (LOADED)\n# Default: 0 (means that it is turned off)\ndropout_rate: 0\n\n# Size of the latent space (LOADED)\nlatent_size: 256\n\n# Number of attention heads (LOADED)\nnum_attention_heads: 4\n\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of encoded questions (INPUT)\n question_encodings: question_encodings\n\n # Stream containing outputs (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the question encodings input (RETRIEVED)\n question_encoding_size: question_encoding_size\n\n # Size of the output (RETRIEVED)\n output_size: output_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/models/vision/convnet_encoder.yml", "content": "# This file defines the default values for the simple 3-layer ConvNet model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Parameters defining the first convolutional layer (LOADED)\nconv1:\n out_channels: 64\n kernel_size: 3\n stride: 1\n padding: 0\n\n# Parameters defining the first max-pooling layer (LOADED)\nmaxpool1:\n kernel_size: 2\n\n# Parameters defining the second convolutional layer (LOADED)\nconv2:\n out_channels: 32\n kernel_size: 3\n stride: 1\n padding: 0\n\n# Parameters defining the second max-pooling layer (LOADED)\nmaxpool2:\n kernel_size: 2\n\n# Parameters defining the third convolutional layer (LOADED)\nconv3:\n out_channels: 16\n kernel_size: 3\n stride: 1\n padding: 0\n\n# Parameters defining the third max-pooling layer (LOADED)\nmaxpool3:\n kernel_size: 2\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of inputs (INPUT)\n inputs: inputs\n\n # Stream containing batch of feature maps (OUTPUT)\n feature_maps: feature_maps\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Height dimension of the input (RETRIEVED)\n input_height: image_height\n\n # Width dimension of the image (RETRIEVED)\n input_width: image_width\n\n # Depth dimension of the image (RETRIEVED)\n input_depth: image_depth\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Height of the feature map (SET)\n feature_map_height: feature_map_height\n\n # Width of the feature map (SET)\n feature_map_width: feature_map_width\n\n # Depth of the feature map (SET)\n feature_map_depth: feature_map_depth\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/models/vision/generic_image_encoder.yml", "content": "# This file defines the default values for the component wrapping (pretrained) Torch Vision models.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Model type (LOADED)\n# Options: vgg16 | densenet121 | resnet152 | resnet50\nmodel_type: vgg16\n\n# Parameter denoting whether the component will return (flat) prediction\n# or output of last feature layer (LOADED)\nreturn_feature_maps: False\n\n# Load weights of a model pretrained on ImageNet (LOADED)\npretrained: True\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of images (INPUT)\n inputs: inputs\n\n # Stream containing outputs (features or \"predictions\") (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the output (RETRIEVED)\n # Used when return_features = False.\n output_size: output_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Height of the returned features tensor (SET)\n # Used when return_features = True.\n feature_maps_height: feature_maps_height\n\n # Width of the returned features tensor (SET)\n # Used when return_features = True.\n feature_maps_width: feature_maps_width\n\n # Depth of the returned features tensor (SET)\n # Used when return_features = True.\n feature_maps_depth: feature_maps_depth\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/models/vision/lenet5.yml", "content": "# This file defines the default values for the LeNet5 model.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of images (INPUT)\n inputs: inputs\n\n # Stream containing predictions (OUTPUT)\n predictions: predictions\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the prediction (RETRIEVED)\n prediction_size: prediction_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/publishers/global_variable_publisher.yml", "content": "# This file defines the default values for the Global Variable Publisher.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# List of keys of variables that will be added to globals (LOADED)\n# an be both list of strings or a single string with comma-separated values.\nkeys: ''\n\n# List of values - must be a single value or a list (LOADED)\nvalues: ''\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/publishers/stream_file_exporter.yml", "content": "# This file defines the default values for the Stream File Exporter.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# List of names of streams that will be displayed (LOADED)\n# Can be string a single name or or comma separated string with list\ninput_streams: ''\n\n# Separator that will be placed between values (LOADED)\nseparator: ','\n\n# Adds additional line to output file enabling Excel to use different separator while loading (LOADED)\nexport_separator_line_to_csv: False\n\n# Adds additional line to output with header (LOADED)\nexport_header_to_csv: False\n\n# Name of the file containing output values (LOADED)\nfilename: 'outputs.txt'\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/statistics/accuracy_statistics.yml", "content": "# This file defines the default values for the Accuracy statistics.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Flag indicating whether prediction are represented as distributions or indices (LOADED)\n# Options: True (expects distribution for each preditions)\n# False (expects indices (max args))\nuse_prediction_distributions: True\n\n# When set to True, performs masking of selected samples from batch (LOADED)\nuse_masking: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing targets (label ids) (INPUT)\n targets: targets\n\n # Stream containing batch of predictions (INPUT)\n predictions: predictions\n\n # Stream containing masks used for masking of selected samples from batch (INPUT)\n masks: masks\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\nstatistics:\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n # Name used for collected statistics (ADDED).\n accuracy: accuracy\n\n\n" }, { "path": "configs/default/components/statistics/batch_size_statistics.yml", "content": "# This file defines the default values for the Batch size statistics.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (INPUT)\n indices: indices\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/statistics/bleu_statistics.yml", "content": "# This file defines the default values for the BLEU statistics.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Flag indicating whether prediction are represented as distributions or indices (LOADED)\n# Options: True (expects distribution for each preditions)\n# False (expects indices (max args))\nuse_prediction_distributions: True\n\n# When set to True, performs masking of selected samples from batch (LOADED)\n# TODO!\n#use_masking: False\n\n# Ignored words - useful for ignoring special tokens\nignored_words: [\"\", \"\"]\n\n# Weights of n-grams used when calculating the score.\nweights: [0.25, 0.25, 0.25, 0.25]\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing targets (label ids) (INPUT)\n targets: targets\n\n # Stream containing batch of predictions (INPUT)\n predictions: predictions\n\n # Stream containing masks used for masking of selected samples from batch (INPUT)\n #masks: masks\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Word mappings used for mappings of predictions/targets into list of words (RERIEVED)\n word_mappings: word_mappings\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\nstatistics:\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n # Name used for collected statistics (ADDED).\n bleu: bleu\n\n\n" }, { "path": "configs/default/components/statistics/precision_recall_statistics.yml", "content": "# This file defines the default values for the PrecisionRecall statistics.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Flag indicating whether prediction are represented as distributions or indices (LOADED)\n# Options: True (expects distribution for each preditions)\n# False (expects indices (max args))\nuse_prediction_distributions: True\n\n# Flag indicating whether confusion matrix will be shown (LOADED)\nshow_confusion_matrix: False\n\n# Flag indicating whether detailed scores for each class will be shown (LOADED)\nshow_class_scores: False\n\n# When set to true, will use the provided word mappings as labels (LOADED)\nuse_word_mappings: False\n\n# When set to True, performs masking of selected samples from batch (LOADED)\nuse_masking: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing targets (label ids) (INPUT)\n targets: targets\n\n # Stream containing batch of predictions (INPUT)\n predictions: predictions\n\n # Stream containing masks used for masking of selected samples from batch (INPUT)\n masks: masks\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Word mappings (optionally) used as labels (RETRIEVED)\n word_mappings: word_mappings\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\nstatistics:\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n # Names used for collected statistics (ADDED).\n precision: precision\n recall: recall\n f1score: f1score\n\n\n" }, { "path": "configs/default/components/tasks/image_text_to_class/clevr.yml", "content": "# This file defines the default values for the CLEVR task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/CLEVR_v1.0'\n\n# Defines the set (split) that will be used (LOADED)\n# Options: training | validation | test | cogent_a_training | cogent_a_validation | cogent_b_validation\nsplit: training\n\n# Flag indicating whether the task will load and return images (LOADED)\nstream_images: True\n\n# Resize parameter (LOADED)\n# When present, resizes the images from original size to [height, width]\n# Depth remains set to 3.\n#resize_image: [height, width]\n\n# Select applied image preprocessing/augmentations (LOADED)\n# Use one (or more) of the affine transformations:\n# none | normalize | all\n# Accepted formats: a,b,c or [a,b,c]\nimage_preprocessing: none\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n images: images\n\n # Stream containing batch of image names (OUTPUT)\n image_ids: image_ids\n\n # Stream containing batch of questions (OUTPUT)\n questions: questions\n\n # Stream containing targets - answers (OUTPUT)\n answers: answers\n\n # Stream containing scene descriptions (OUTPUT)\n #answers: scene_graphs\n\n # Stream containing batch with question type - indices (OUTPUT)\n category_ids: question_type_ids\n\n # Stream containing batch with question type - names (OUTPUT)\n category_names: question_type_names\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n\n # Question type (word-idx) mappings (SET)\n question_type_word_mappings: question_type_word_mappings\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/tasks/image_text_to_class/gqa.yml", "content": "# This file defines the default values for the GQA task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/gqa'\n\n# Defines the set (split) that will be used (LOADED)\n# Options: training_0 | training | validation | test_dev | test | challenge | submission (?)\n# Note: test_dev should be used for validation.\nsplit: training_0\n\n# Flag indicating whether the task will load and return images (LOADED)\nstream_images: True\n\n# Resize parameter (LOADED)\n# When present, resizes the images from original size to [height, width]\n# Depth remains set to 3.\nresize_image: [224, 224]\n\n# Select applied image preprocessing/augmentations (LOADED)\n# Use one (or more) of the affine transformations:\n# none | normalize | all\n# Accepted formats: a,b,c or [a,b,c]\nimage_preprocessing: none\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of sample (original) identifiers (OUTPUT)\n sample_ids: sample_ids\n\n # Stream containing batch of images (OUTPUT)\n images: images\n\n # Stream containing batch of image names (OUTPUT)\n image_ids: image_ids\n\n # Stream containing batch of questions (OUTPUT)\n questions: questions\n\n # Stream containing targets answers (labels) (OUTPUT)\n answers: answers\n\n # Stream containing targets answers consisting of many words (OUTPUT)\n full_answers: full_answers\n\n # Stream containing scene descriptions (OUTPUT)\n #answers: scene_graphs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/tasks/image_text_to_class/vqa_med_2019.yml", "content": "# This file defines the default values for the VQAMED2019 task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/vqa-med'\n\n# Defines the set (split) that will be used (LOADED)\n# Options: training | validation | training_validation | test_answers | test\nsplit: training\n\n# Defines the categoriees that will be used (LOADED)\n# Options: all | c1 | c2 | c3 | c4 (or any combination of the latter 4)\ncategories: all\n\n# Flag indicating whether the task will load and return images (LOADED)\nstream_images: True\n\n# Flag indicating whether images will be preloaded (i.e. loaded once at start) (LOADED)\n# WARNING: if this option is active, the images will also be \"preprocessed\" at start.\n# This means that preloading should not be used when one needs to use the random augmentations!\npreload_images: False\n\n# Resize parameter (LOADED)\n# When present, resizes the images from original size to [height, width]\n# Depth remains set to 3.\n#resize_image: [height, width]\n\n# Scale parameter [height, width] (LOADED)\n# Task will use those values to rescale the image_sizes to range (0, 1).\nscale_image_size: [2414, 2323]\n\n# Select applied image preprocessing/augmentations (LOADED)\n# Use one (or more) of the affine transformations:\n# none | random_affine | random_horizontal_flip | normalize | all\n# Accepted formats: a,b,c or [a,b,c]\nimage_preprocessing: normalize\n\n# Select applied question preprocessing/augmentations (LOADED)\n# Use one (or more) of the transformations:\n# none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n# Accepted formats: a,b,c or [a,b,c]\nquestion_preprocessing: lowercase, remove_punctuation\n\n# Select applied question preprocessing (LOADED)\n# Use one (or more) of the transformations:\n# none | lowercase | remove_punctuation | tokenize | all\n# Accepted formats: a,b,c or [a,b,c]\nanswer_preprocessing: none\n\n# When filename is not empty, task will calculate weights associated with all samples\n# by looking at the distribution of all answers from all loaded samples (LOADED)\n# Those weights can be next used by weighted samplers (e.g. kFoldWeightedSampler)\nexport_sample_weights: ''\n\n# Shuffle the indices of the input (source) files/samples.\n# Leaving that to false will results in the original order of files samples,\n# i.e. C1, then C2, then C3 etc.\nshuffle_indices: False\n\n# Generate and export (potentially shuffled) indices (LOADED)\n# If not empty, will:\n# * shuffle indices of all samples and export them to a file.\n# * use those indices during sampling.\nexport_indices: ''\n\n# Import (potentially shuffled) indices (LOADED)\n# If not empty, will:\n# * import them to a file.\n# * use those indices during sampling.\nimport_indices: ''\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n images: images\n\n # Stream containing batch of image names (OUTPUT)\n image_ids: image_ids\n\n # Stream containing batch with original sizes of images (OUTPUT)\n image_sizes: image_sizes\n\n # Stream containing batch of questions (OUTPUT)\n questions: questions\n\n # Stream containing targets - answers (OUTPUT)\n answers: answers\n\n # Stream containing batch with question categories - indices (OUTPUT)\n category_ids: category_ids\n\n # Stream containing batch with question categories - names (OUTPUT)\n category_names: category_names\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n # Category (word-idx) mappings (SET)\n category_word_mappings: category_word_mappings\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n" }, { "path": "configs/default/components/tasks/image_to_class/cifar_100.yml", "content": "# This file defines the default values for the CIFAR-100 task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/cifar-100'\n\n# Defines the set that will be used used (LOADED)\n# True: training set | False: test set.\nuse_train_data: True\n\n# Optional parameter (LOADED)\n# When present, resizes the CIFAR images from [32,32] to [width, height]\n#resize_image: [height, width]\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n images: images\n\n # Streams containing targets (label ids) (OUTPUT)\n coarse_targets: coarse_targets\n fine_targets: fine_targets\n\n # Streams containing labels (words) (OUTPUT)\n coarse_labels: coarse_labels\n fine_labels: fine_labels\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n \n # Numbers of output classes (SET)\n coarse_num_classes: coarse_num_classes\n fine_num_classes: fine_num_classes\n\n # Labels (word-idx) mappings (SET)\n coarse_label_word_mappings: coarse_label_word_mappings\n fine_label_word_mappings: fine_label_word_mappings\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/image_to_class/mnist.yml", "content": "# This file defines the default values for the MNIST task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/mnist'\n\n# Defines the set that will be used used (LOADED)\n# True: training set | False: test set.\nuse_train_data: True\n\n# Optional parameter (LOADED)\n# When present, resizes the MNIST images from [28,28] to [width, height]\n#resize_image: [height, width]\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n inputs: inputs\n\n # Stream containing targets (label ids) (OUTPUT)\n targets: targets\n\n # Stream containing labels (words) (OUTPUT)\n labels: labels\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n \n # Number of output classes: 10 (SET)\n num_classes: num_classes\n # Label (word-idx) mappings (SET)\n label_word_mappings: label_word_mappings\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/image_to_class/simple_molecules.yml", "content": "# This file defines the default values for the MNIST task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/simple-molecules'\n\n# Defines the split that will be used used (LOADED)\n# Options: training | validation | test | test_mirror_blur | test_on_grid | test_handwritten\nsplit: training\n\n# Optional parameter (LOADED)\n# When present, resizes the MNIST images from [28,28] to [width, height]\n#resize_image: [height, width]\n\n# Depth of the retured image (LOADED)\n# Options: 1 | 3 \nimage_depth: 1\n\nstreams:\n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n images: images\n\n # Stream containing targets (label ids) (OUTPUT)\n targets: targets\n\n # Stream containing labels (words) (OUTPUT)\n labels: labels\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Width of the image (SET)\n input_width: image_width\n # Height of the image (SET)\n input_height: image_height\n # Depth of the image (SET)\n input_depth: image_depth\n \n # Number of output classes: 10 (SET)\n num_classes: num_classes\n # Label (word-idx) mappings (SET)\n label_word_mappings: label_word_mappings\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/text_to_class/dummy_language_identification.yml", "content": "# This file defines the default values for the dummy language identification task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/language_identification/dummy'\n\n# Defines the set that will be used used (LOADED)\n# True: training set | False: test set.\nuse_train_data: True\n\n# If set, task will always (re)generate data (LOADED)\nregenerate: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n inputs: inputs\n\n # Stream containing targets (label ids) (OUTPUT)\n targets: targets\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/text_to_class/wily_language_identification.yml", "content": "# This file defines the default values for the WiLY language identification task.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/language_identification/wily'\n\n# Defines the set that will be used used (LOADED)\n# True: training set | False: test set.\nuse_train_data: True\n\n# If set, task will always (re)generate data (LOADED)\nregenerate: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n inputs: inputs\n\n # Stream containing targets (label ids) (OUTPUT)\n targets: targets\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/text_to_class/wily_ngram_language_modeling.yml", "content": "# This file defines the default values for the ngram language modeling\n# using WiLY dataset.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: '~/data/language_identification/wily'\n\n# Defines the set that will be used used (LOADED)\n# True: training set | False: test set.\nuse_train_data: True\n\n# Size of the context (LOADED)\ncontext: 2\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of images (OUTPUT)\n inputs: inputs\n\n # Stream containing targets (label ids) (OUTPUT)\n targets: targets\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/text_to_text/translation_pairs.yml", "content": "# This file defines the default values for the WikiText language modeling.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: ~/data/language_modeling/translation_pairs\n\n# Defines the dataset that will be used used (LOADED)\n# Options: eng-fra, eng-pol\ndataset: eng-fra\n\n# Defines the used subset (LOADED)\n# Options: train | valid | test\nsubset: train\n\n# Length limit of source and target sentence\n# if < 0, no limit\nsentence_length: 10\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of tokenized source sentences (OUTPUT)\n sources: sources\n\n # Stream containing batch of tokenized target sentences (OUTPUT)\n targets: targets\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/tasks/text_to_text/wikitext_language_modeling.yml", "content": "# This file defines the default values for the WikiText language modeling.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Folder where task will store data (LOADED)\ndata_folder: ~/data/language_modeling/wikitext\n\n# Defines the dataset that will be used used (LOADED)\n# Options: wikitext-2 | wikitext-103\ndataset: wikitext-2\n\n# Defines the used subset (LOADED)\n# Options: train | valid | test\nsubset: train\n\n# Length of sentence (i.e. number of tokens in input and target sentences)\nsentence_length: 50\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing batch of indices (OUTPUT)\n # Every task MUST return that stream.\n indices: indices\n\n # Stream containing batch of tokenized source sentences (OUTPUT)\n sources: sources\n\n # Stream containing batch of tokenized target sentences (OUTPUT)\n targets: targets\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/transforms/concatenate_tensor.yml", "content": "# This file defines the default values for the ConcatenateTensor.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# List of names of streams that will be concatenated (LOADED)\n# Can be string a single name or or comma separated string with list\ninput_streams: ''\n\n# Dimensions of the input tensors (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# List of lists.\n# input_dims: [[BATCH x ...], ...]\n\n# Dimensions of the input tensor (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# output_dims: [BATCH x ...]\n\n# Dimension along which tensors will be concatenated (LOADED)\ndim: 1\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output tensor (SET)\n # (all dimensions except batch_size)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/transforms/list_to_tensor.yml", "content": "# This file defines the default values for the List to Tensor transformation.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Number of input dimensions, including last tensor (LOADED)\nnum_inputs_dims: 1\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the intput_item (GET)\n # (last dimenstion)\n input_size: input_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/transforms/non_linearity.yml", "content": "# This file defines the default values for the NonLinearity.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dimensions of the input tensor (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# input_dims: [BATCH x ....]\n\n# Dimensions of the input tensor (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# output_dims: [BATCH x ....]\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output tensor (SET)\n # (all dimensions except batch_size)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/transforms/reduce_tensor.yml", "content": "# This file defines the default values for the ReduceTensor transformation.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Number of input dimensions, including batch (LOADED)\nnum_inputs_dims: 2\n\n# Dimension along with the reduction will be applied (LOADED)\nreduction_dim: 1\n\n# Reduction type (LOADED)\n# Options: sum | mean | min | max | argmin | argmax\nreduction_type: sum\n\n# If True, the output tensor is of the same size as input, except dim where it is of size 1 (LOADED)\nkeepdim: False\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n # Size of the intput_item (GET)\n # (last dimenstion)\n input_size: input_size\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/transforms/reshape_tensor.yml", "content": "# This file defines the default values for the Tensor Reshaper.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Dimensions of the input tensor (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# input_dims: [BATCH x ....]\n\n# Dimensions of the input tensor (LOADED)\n# Specific to a task/model, must be set by user (no DEFAULT)\n# output_dims: [BATCH x ....]\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing input tensor (INPUT)\n inputs: inputs\n\n # Stream containing output tensor (OUTPUT)\n outputs: outputs\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n # Size of the output tensor (SET)\n # (all dimensions except batch_size)\n output_size: output_size\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/viewers/image_viewer.yml", "content": "# This file defines the default values for the ImageViewer.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# Number of sample that will be printed (LOADED)\n# Default: -1 (means random)\nsample_number: -1\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\n # Stream containing inages (INPUT)\n images: images\n\n # Stream containing target labels (strings) (INPUT)\n label: labels\n\n # Stream containing predicted labels (strings) (INPUT)\n answers: answers\n\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/components/viewers/stream_viewer.yml", "content": "# This file defines the default values for the Stream Viewer.\n\n####################################################################\n# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.\n####################################################################\n\n# List of names of streams that will be displayed (LOADED)\n# Can be string a single name or or comma separated string with list\ninput_streams: ''\n\n# Number of sample that will be printed (LOADED)\n# Default: -1 (means random)\nsample_number: -1\n\nstreams: \n ####################################################################\n # 2. Keymappings associated with INPUT and OUTPUT streams.\n ####################################################################\n\nglobals:\n ####################################################################\n # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.\n ####################################################################\n\n ####################################################################\n # 4. Keymappings associated with GLOBAL variables that will be SET.\n ####################################################################\n\n ####################################################################\n # 5. Keymappings associated with statistics that will be ADDED.\n ####################################################################\n\n" }, { "path": "configs/default/workers/offline_trainer.yml", "content": "####################################################################\n# Section defining all the default values of parameters used during training when using ptp-offline-trainer.\n# If you want to use different section for \"training\" pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)\n# Note: the following parameters will be (anyway) used as default values.\ndefault_training:\n # Set the random seeds: -1 means that they will be picked randomly.\n # Note: their final values will be stored in the final training_configuration.yml saved to log dir.\n seed_numpy: -1\n seed_torch: -1\n\n # Default batch size.\n batch_size: 64\n\n # Definition of the task (Mandatory!)\n #task:\n # One must define its type (Mandatory!)\n # type: ?\n # The rest of the content of that section is task-specific...\n \n # Section describing curriculum learning (Optional)\n #curriculum_learning: \n # # Flag indicating whether curriculum learning has to finish before (eventual) termination of the training.\n # must_finish: True\n # The rest of the content of that section is task-specific...\n\n # Definition of optimizer (Mandatory!)\n #optimizer:\n # # Type - generally all optimizers from PyTorch.optim are allowed (Mandatory!)\n # type: Adam\n # # Options: \n # lr: 0.0001\n # The rest of the content of that section is optimizer-specific...\n\n # Set a default configuration section for data loader.\n dataloader:\n # Shuffle set by default.\n shuffle: True \n batch_sampler: None\n # Do not use multiprocessing by default.\n num_workers: 0\n pin_memory: False\n # Do not drop last frame by default.\n drop_last: False\n timeout: 0\n\n # Definition of sampler (Optional)\n # When this section will not be present, worker will use \"standard\" sampling (please refer to shuffle in dataloader)\n #sampler:\n # # Type - generally all samplers from PyTorch (plus some new onses) are allowed (Mandatory!)\n # # Options: \n # type: RandomSmpler\n # The rest of the content of that section is optimizer-specific...\n\n # Terminal conditions that will be used during training.\n # They can (and ofter should) be overwritten.\n terminal_conditions:\n # Terminal condition I: loss threshold, going below will terminate the training.\n loss_stop_threshold: 0.00001 # 1e-5\n # Terminal condition II: Early stopping monitor validation loss, if it didn't down during last n validations, training will be terminated (Optional, negative means that this condition is disabled)\n early_stop_validations: 10\n # Terminal condition III: maximal number of epochs (Mandatory for this trainer! Must be > 0)\n epoch_limit: 10\n # Terminal condition IV: maximal number of episodes (Optional, -1 (negative) means that this condition is disabled)\n episode_limit: -1\n\n\n\n####################################################################\n# Section defining all the default values of parameters used during validation.\n# If you want to use different section for validation pass its name as command line argument '--validation_section_name' to trainer (DEFAULT: validation)\n# Note: the following parameters will be (anyway) used as default values.\ndefault_validation:\n # Defines how often the partial validation will be performed.\n # In this trainer Partial Validation is optional (negative value means it is disabled)\n partial_validation_interval: -1\n\n # Definition of the task (mandatory!)\n #task:\n # One must define its type (Mandatory!)\n # type: ?\n # The rest of the content of that section is task-specific...\n\n # Set a default configuration section for data loader.\n dataloader:\n # Shuffle set by default.\n shuffle: True \n # Do not use multiprocessing by default.\n num_workers: 0\n pin_memory: False\n # Do not drop last frame by default.\n drop_last: False\n timeout: 0\n\n # Definition of sampler (Optional)\n # When this section will not be present, worker will use \"standard\" sampling (please refer to shuffle in dataloader)\n #sampler:\n # # Type - generally all samplers from PyTorch (plus some new onses) are allowed (Mandatory!)\n # # Options: \n # type: RandomSmpler\n # The rest of the content of that section is optimizer-specific...\n\n\n\n####################################################################\n# Section defining all the default values of parameters used during training.\n# If you want to use different section for validation pass its name as command line argument '--pipeline_section_name' to trainer (DEFAULT: pipeline)\npipeline: \n # Pipeline must contain at least one component.\n #name_1:\n # Each component must have defined its priority... (Mandatory!)\n # priority: 0.1 # Can be float. Smaller means higher priority, up to zero.\n # # ... and type (Mandatory!)\n # type: ?\n # The rest of the content of that section is component-specific...\n\n\n" }, { "path": "configs/default/workers/online_trainer.yml", "content": "####################################################################\n# Section defining all the default values of parameters used during training when using ptp-online-trainer.\n# If you want to use different section for \"training\" pass its name as command line argument '--training_section_name' to trainer (DEFAULT: training)\n# Note: the following parameters will be (anyway) used as default values.\ndefault_training:\n # Set the random seeds: -1 means that they will be picked randomly.\n # Note: their final values will be stored in the final training_configuration.yml saved to log dir.\n seed_numpy: -1\n seed_torch: -1\n\n # Default batch size.\n batch_size: 64\n\n # Definition of the task (Mandatory!)\n #task:\n # One must define its type (Mandatory!)\n # type: ?\n # The rest of the content of that section is task-specific...\n \n # Section describing curriculum learning (Optional)\n #curriculum_learning: \n # # Flag indicating whether curriculum learning has to finish before (eventual) termination of the training.\n # must_finish: True\n # The rest of the content of that section is task-specific...\n\n # Definition of optimizer (Mandatory!)\n #optimizer:\n # # Type - generally all optimizers from PyTorch.optim are allowed (Mandatory!)\n # type: Adam\n # # Options: \n # lr: 0.0001\n # The rest of the content of that section is optimizer-specific...\n\n # Set a default configuration section for data loader.\n dataloader:\n # Shuffle set by default.\n shuffle: True \n batch_sampler: None\n # Do not use multiprocessing by default.\n num_workers: 0\n pin_memory: False\n # Do not drop last frame by default.\n drop_last: False\n timeout: 0\n\n # Definition of sampler (Optional)\n # When this section will not be present, worker will use \"standard\" sampling (please refer to shuffle in dataloader)\n #sampler:\n # # Type - generally all samplers from PyTorch (plus some new onses) are allowed (Mandatory!)\n # # Options: \n # type: RandomSmpler\n # The rest of the content of that section is optimizer-specific...\n\n # Terminal conditions that will be used during training.\n # They can (and ofter should) be overwritten.\n terminal_conditions:\n # Terminal condition I: loss threshold, going below will terminate the training.\n loss_stop_threshold: 0.00001 # 1e-5\n # Terminal condition II: Early stopping monitor validation loss, if it didn't down during last n validations, training will be terminated (Optional, negative means that this condition is disabled)\n early_stop_validations: 10\n # Terminal condition III: maximal number of epochs (Optional, -1 (negative) means that this condition is disabled)\n epoch_limit: -1\n # Terminal condition IV: maximal number of episodes (Mandatory for this trainer! Must be > 0)\n episode_limit: 100000\n\n\n\n####################################################################\n# Section defining all the default values of parameters used during validation.\n# If you want to use different section for validation pass its name as command line argument '--validation_section_name' to trainer (DEFAULT: validation)\n# Note: the following parameters will be (anyway) used as default values.\ndefault_validation:\n # Defines how often the partial validation will be performed.\n # In this trainer Partial Validation is mandatory, hence interval must be > 0.\n partial_validation_interval: 100\n\n # Definition of the task (mandatory!)\n #task:\n # One must define its type (Mandatory!)\n # type: ?\n # The rest of the content of that section is task-specific...\n\n # Set a default configuration section for data loader.\n dataloader:\n # Shuffle set by default.\n shuffle: True \n # Do not use multiprocessing by default.\n num_workers: 0\n pin_memory: False\n # Do not drop last frame by default.\n drop_last: False\n timeout: 0\n\n # Definition of sampler (Optional)\n # When this section will not be present, worker will use \"standard\" sampling (please refer to shuffle in dataloader)\n #sampler:\n # # Type - generally all samplers from PyTorch (plus some new onses) are allowed (Mandatory!)\n # # Options: \n # type: RandomSmpler\n # The rest of the content of that section is optimizer-specific...\n\n\n\n####################################################################\n# Section defining all the default values of parameters used during training.\n# If you want to use different section for validation pass its name as command line argument '--pipeline_section_name' to trainer (DEFAULT: pipeline)\npipeline: \n # Pipeline must contain at least one component.\n #name_1:\n # Each component must have defined its priority... (Mandatory!)\n # priority: 0.1 # Can be float. Smaller means higher priority, up to zero.\n # # ... and type (Mandatory!)\n # type: ?\n # The rest of the content of that section is component-specific...\n\n\n" }, { "path": "configs/default/workers/processor.yml", "content": "####################################################################\n# Section defining all the default values of parameters used during testing.\n# If you want to use different section during \"processing\" pass its name as command line argument '--section_name' to trainer (DEFAULT: test)\n# Note: the following parameters will be (anyway) used as default values.\ndefault_test:\n # Set the random seeds: -1 means that they will be picked randomly.\n seed_numpy: -1\n seed_torch: -1\n\n # Default batch size.\n batch_size: 64\n\n # Definition of the task (Mandatory!)\n #task:\n # One must define its type (Mandatory!)\n # type: ?\n # The rest of the content of that section is task-specific...\n\n # Set a default configuration section for data loader.\n dataloader:\n # Shuffle set by default.\n shuffle: True \n batch_sampler: None\n # Do not use multiprocessing by default.\n num_workers: 0\n pin_memory: False\n # Do not drop last frame by default.\n drop_last: False\n timeout: 0\n\n # Definition of sampler (Optional)\n # When this section will not be present, worker will use \"standard\" sampling (please refer to shuffle in dataloader)\n #sampler:\n # # Type - generally all samplers from PyTorch (plus some new onses) are allowed (Mandatory!)\n # # Options: \n # type: RandomSampler\n # The rest of the content of that section is optimizer-specific...\n\n # Terminal condition that will be used during processing.\n terminal_conditions:\n # Terminal condition : maximal number of episodes (Optional, -1 means that processor will perform one pass over the whole dataset/split)\n episode_limit: -1\n\n\n####################################################################\n# Section defining all the default values of parameters used during training.\n# If you want to use different section for validation pass its name as command line argument '--pipeline_section_name' to trainer (DEFAULT: pipeline)\npipeline: \n # Pipeline must contain at least one component.\n #name_1:\n # Each component must have defined its priority... (Mandatory!)\n # priority: 0.1 # Can be float. Smaller means higher priority, up to zero.\n # # ... and type (Mandatory!)\n # type: ?\n # The rest of the content of that section is component-specific...\n" }, { "path": "configs/mnist/default_mnist.yml", "content": "# Training parameters:\ntraining:\n task: \n type: MNIST\n batch_size: &b 64\n use_train_data: True\n #resize: [32, 32]\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [0, 55000]\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.05\n early_stop_validations: -1\n episode_limit: 10000\n epoch_limit: 10\n\n# Validation parameters:\nvalidation:\n #partial_validation_interval: 100\n task:\n type: MNIST\n batch_size: *b\n use_train_data: True # True because we are splitting the training set to: validation and training\n #resize: [32, 32]\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [55000, 60000]\n\n# Testing parameters:\ntest:\n task:\n type: MNIST\n batch_size: *b\n use_train_data: False\n #resize: [32, 32]\n\npipeline:\n disable: image_viewer\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.0\n\n # Statistics.\n batch_size:\n priority: 100.0\n type: BatchSizeStatistics\n\n accuracy:\n priority: 100.1\n type: AccuracyStatistics\n\n\n precision_recall:\n priority: 100.2\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n globals:\n word_mappings: label_word_mappings\n\n answer_decoder:\n priority: 100.3\n type: WordDecoder\n import_word_mappings_from_globals: True\n globals:\n word_mappings: label_word_mappings\n streams:\n inputs: predictions\n outputs: answers\n\n stream_viewer:\n priority: 100.4\n type: StreamViewer\n input_streams: labels, answers\n\n image_viewer:\n priority: 100.5\n type: ImageViewer\n streams:\n images: inputs\n labels: labels\n answers: answers\n\n" }, { "path": "configs/mnist/mnist_classification_convnet_softmax.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\npipeline:\n\n # Model consisting of two components.\n image_encoder:\n type: ConvNetEncoder\n priority: 1\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 1, 1]\n output_dims: [-1, 16]\n priority: 2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Image classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 3\n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/mnist/mnist_classification_kfold_softmax.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\n# Training parameters:\ntraining:\n task: \n type: MNIST\n batch_size: &b 64\n use_train_data: True\n # Use k-fold cross-validation random sampler.\n sampler:\n type: kFoldRandomSampler\n folds: 10 # Each with size of 6000\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.5\n episode_limit: 10000\n epoch_limit: 5\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 500\n task:\n type: MNIST\n batch_size: *b\n use_train_data: True # True because we are splitting the training set to: validation and training\n # Use k-fold cross-validation random sampler.\n sampler:\n type: kFoldRandomSampler\n folds: 10 # Each with size of 6000\n\npipeline:\n\n # Reshapes tensors.\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 1, 28, 28]\n output_dims: [-1, 784]\n priority: 1\n streams:\n outputs: reshaped_images\n globals:\n output_size: reshaped_image_size\n\n # Classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 2\n dropout_rate: 0.1\n hidden_sizes: [100, 100]\n streams:\n inputs: reshaped_images\n globals:\n input_size: reshaped_image_size\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/mnist/mnist_classification_lenet5.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\n# Training parameters - overwrite defaults:\ntraining:\n task: \n resize_image: [32, 32]\n\n# Validation parameters - overwrite defaults:\nvalidation:\n task:\n resize_image: [32, 32]\n\n# Testing parameters - overwrite defaults:\ntest:\n task:\n resize_image: [32, 32]\n\n# Definition of the pipeline.\npipeline:\n\n # Image classifier.\n image_classifier:\n type: LeNet5\n priority: 1\n globals:\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/mnist/mnist_classification_softmax.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\npipeline:\n\n # Reshapes tensors.\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 1, 28, 28]\n output_dims: [-1, 784]\n priority: 1\n streams:\n outputs: reshaped_images\n globals:\n output_size: reshaped_image_size\n\n # Classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 2\n dropout_rate: 0.1\n hidden_sizes: [100, 100]\n streams:\n inputs: reshaped_images\n globals:\n input_size: reshaped_image_size\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/mnist/mnist_classification_vf_2lenet5_2losses.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\n# Training parameters - overwrite defaults:\ntraining:\n task: \n resize_image: [32, 32]\n\n# Validation parameters - overwrite defaults:\nvalidation:\n task:\n resize_image: [32, 32]\n\n# Testing parameters - overwrite defaults:\ntest:\n task:\n resize_image: [32, 32]\n\n# Definition of the pipeline.\npipeline:\n\n # Disable components for \"default\" flow.\n disable: nllloss, accuracy, precision_recall, image_viewer\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n keys: [num_classes1, num_classes2, word_to_ix1, word_to_ix2]\n values: [3, 7, {\"Zero\": 0, \"One\": 1, \"Two\": 2}, {\"Three\": 0, \"Four\": 1, \"Five\": 2, \"Six\": 3, \"Seven\": 4, \"Eight\": 5, \"Nine\": 6}]\n\n ################# Flow 1 #################\n # Image classifier.\n flow1_image_classifier:\n type: LeNet5\n priority: 1.1\n globals:\n prediction_size: num_classes1\n streams:\n inputs: inputs\n predictions: flow1_predictions\n \n flow1_label_to_mask1:\n type: StringToMask\n priority: 1.2\n globals:\n word_mappings: word_to_ix1\n streams:\n strings: labels\n masks: flow1_masks\n\n flow1_label_to_target1:\n type: LabelIndexer\n priority: 1.3\n import_word_mappings_from_globals: True\n globals:\n word_mappings: word_to_ix1\n streams:\n inputs: labels\n outputs: flow1_targets\n\n # Masked loss.\n flow1_nllloss:\n type: NLLLoss\n priority: 1.4\n use_masking: True\n streams:\n targets: flow1_targets\n predictions: flow1_predictions\n masks: flow1_masks\n loss: flow1_loss\n\n # Statistics.\n flow1_accuracy:\n type: AccuracyStatistics\n priority: 1.51\n use_masking: True\n streams:\n predictions: flow1_predictions\n targets: flow1_targets\n masks: flow1_masks\n statistics:\n accuracy: flow1_accuracy\n\n flow1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 1.52\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n use_masking: True\n globals:\n word_mappings: word_to_ix1\n num_classes: num_classes1\n streams:\n targets: flow1_targets\n predictions: flow1_predictions\n masks: flow1_masks\n statistics:\n precision: flow1_precision\n recall: flow1_recall\n f1score: flow1_f1score\n\n ################# Flow 2 #################\n # Image classifier.\n flow2_image_classifier:\n type: LeNet5\n priority: 2.1\n globals:\n prediction_size: num_classes2\n streams:\n inputs: inputs\n predictions: flow2_predictions\n \n flow2_label_to_mask2:\n type: StringToMask\n priority: 2.2\n globals:\n word_mappings: word_to_ix2\n streams:\n strings: labels\n masks: flow2_masks\n\n flow2_label_to_target2:\n type: LabelIndexer\n priority: 2.3\n import_word_mappings_from_globals: True\n globals:\n word_mappings: word_to_ix2\n streams:\n inputs: labels\n outputs: flow2_targets\n\n # Masked loss.\n flow2_nllloss:\n type: NLLLoss\n priority: 2.4\n use_masking: True\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n loss: flow2_loss\n\n # Statistics.\n flow2_accuracy:\n type: AccuracyStatistics\n priority: 2.41\n use_masking: True\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n statistics:\n accuracy: flow2_accuracy\n\n flow2_precision_recall:\n type: PrecisionRecallStatistics\n priority: 2.42\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n use_masking: True\n globals:\n word_mappings: word_to_ix2\n num_classes: num_classes2\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n statistics:\n precision: flow2_precision\n recall: flow2_recall\n f1score: flow2_f1score\n\n ################# JOIN #################\n joined_predictions:\n type: JoinMaskedPredictions\n priority: 3.1\n # Names of used input streams.\n input_prediction_streams: [flow1_predictions, flow2_predictions]\n input_mask_streams: [flow1_masks, flow2_masks]\n input_word_mappings: [word_to_ix1, word_to_ix2]\n globals:\n output_word_mappings: label_word_mappings # from MNIST task.\n streams:\n output_strings: merged_predictions\n output_indices: merged_indices\n\n # Statistics.\n joined_accuracy:\n type: AccuracyStatistics\n priority: 3.21\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n streams:\n targets: targets\n predictions: merged_indices\n statistics:\n accuracy: joined_accuracy\n\n joined_precision_recall:\n type: PrecisionRecallStatistics\n priority: 3.22\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: label_word_mappings # straight from MNIST\n #num_classes: num_classes\n streams:\n targets: targets # straight from MNIST\n predictions: merged_indices\n statistics:\n precision: joined_precision\n recall: joined_recall\n f1score: joined_f1score\n\n # \"Fix\" (overwrite) stream names in viewers.\n image_viewer:\n streams:\n answers: merged_predictions\n\n stream_viewer:\n input_streams: labels, merged_predictions\n\n#: pipeline\n" }, { "path": "configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: mnist/default_mnist.yml\n\n# Training parameters - overwrite defaults:\ntraining:\n task: \n #resize_image: [32, 32]\n batch_size: 64\n #optimizer:\n # #type: Adam\n # lr: 0.001\n #terminal_conditions:\n # loss_stop_threshold: 0.08\n\n# Validation parameters - overwrite defaults:\n#validation:\n# partial_validation_interval: 10\n# task:\n# resize_image: [32, 32]\n\n# Testing parameters - overwrite defaults:\n#test:\n# task:\n# resize_image: [32, 32]\n\n# Definition of the pipeline.\npipeline:\n\n # Disable components for \"default\" flow.\n disable: nllloss, accuracy, precision_recall, answer_decoder, image_viewer\n\n ################# SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0.1\n keys: [num_classes1, num_classes2, word_to_ix1, word_to_ix2]\n values: [3, 7, {\"Three\": 0, \"One\": 1, \"Five\": 2}, {\"Four\": 0, \"Two\": 1, \"Zero\": 2, \"Six\": 3, \"Seven\": 4, \"Eight\": 5, \"Nine\": 6}]\n #values: [3, 7, {\"Zero\": 0, \"One\": 1, \"Two\": 2}, {\"Three\": 0, \"Four\": 1, \"Five\": 2, \"Six\": 3, \"Seven\": 4, \"Eight\": 5, \"Nine\": 6}]\n\n # Shared model - encoder.\n image_encoder:\n type: ConvNetEncoder\n priority: 0.2\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 1, 1]\n output_dims: [-1, 16]\n priority: 0.3\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n ################# Flow 1 #################\n # Classifier.\n flow1_classifier:\n type: FeedForwardNetwork \n priority: 1.1\n streams:\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_classes1\n streams:\n inputs: reshaped_maps\n predictions: flow1_predictions\n \n flow1_label_to_mask1:\n type: StringToMask\n priority: 1.2\n globals:\n word_mappings: word_to_ix1\n streams:\n strings: labels\n masks: flow1_masks\n\n flow1_label_to_target1:\n type: LabelIndexer\n priority: 1.3\n import_word_mappings_from_globals: True\n globals:\n word_mappings: word_to_ix1\n streams:\n inputs: labels\n outputs: flow1_targets\n\n # Masked loss.\n flow1_nllloss:\n type: NLLLoss\n priority: 1.4\n use_masking: True\n streams:\n targets: flow1_targets\n predictions: flow1_predictions\n masks: flow1_masks\n loss: flow1_loss\n\n # Statistics.\n flow1_accuracy:\n type: AccuracyStatistics\n priority: 1.51\n use_masking: True\n streams:\n predictions: flow1_predictions\n targets: flow1_targets\n masks: flow1_masks\n statistics:\n accuracy: flow1_accuracy\n\n flow1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 1.52\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n use_masking: True\n globals:\n word_mappings: word_to_ix1\n num_classes: num_classes1\n streams:\n targets: flow1_targets\n predictions: flow1_predictions\n masks: flow1_masks\n statistics:\n precision: flow1_precision\n recall: flow1_recall\n f1score: flow1_f1score\n\n ################# Flow 2 #################\n # Classifier.\n flow2_classifier:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_classes2\n streams:\n inputs: reshaped_maps\n predictions: flow2_predictions\n \n flow2_label_to_mask2:\n type: StringToMask\n priority: 2.2\n globals:\n word_mappings: word_to_ix2\n streams:\n strings: labels\n masks: flow2_masks\n\n flow2_label_to_target2:\n type: LabelIndexer\n priority: 2.3\n import_word_mappings_from_globals: True\n globals:\n word_mappings: word_to_ix2\n streams:\n inputs: labels\n outputs: flow2_targets\n\n # Masked loss.\n flow2_nllloss:\n type: NLLLoss\n priority: 2.4\n use_masking: True\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n loss: flow2_loss\n\n # Statistics.\n flow2_accuracy:\n type: AccuracyStatistics\n priority: 2.41\n use_masking: True\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n statistics:\n accuracy: flow2_accuracy\n\n flow2_precision_recall:\n type: PrecisionRecallStatistics\n priority: 2.42\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n use_masking: True\n globals:\n word_mappings: word_to_ix2\n num_classes: num_classes2\n streams:\n targets: flow2_targets\n predictions: flow2_predictions\n masks: flow2_masks\n statistics:\n precision: flow2_precision\n recall: flow2_recall\n f1score: flow2_f1score\n\n ################# JOIN #################\n joined_predictions:\n type: JoinMaskedPredictions\n priority: 3.1\n # Names of used input streams.\n input_prediction_streams: [flow1_predictions, flow2_predictions]\n input_mask_streams: [flow1_masks, flow2_masks]\n input_word_mappings: [word_to_ix1, word_to_ix2]\n globals:\n output_word_mappings: label_word_mappings # from MNIST task.\n streams:\n output_strings: merged_predictions\n output_indices: merged_indices\n\n # Statistics.\n joined_precision_recall:\n type: PrecisionRecallStatistics\n priority: 3.22\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: label_word_mappings # straight from MNIST\n #num_classes: num_classes\n streams:\n targets: targets # straight from MNIST\n predictions: merged_indices\n statistics:\n precision: joined_precision\n recall: joined_recall\n f1score: joined_f1score\n\n # \"Fix\" (overwrite) stream names in viewers.\n image_viewer:\n streams:\n answers: merged_predictions\n\n stream_viewer:\n input_streams: labels, merged_predictions\n\n#: pipeline\n" }, { "path": "configs/molecule_classification/default_molecule_classification.yml", "content": "# Training parameters:\ntraining:\n task: \n type: SimpleMolecules\n batch_size: &b 64\n split: training\n resize_image: [87, 87]\n # Resizing slows down the batch generation, so let's use many dataloaders.\n dataloader:\n num_workers: 4\n\n optimizer:\n type: Adam\n lr: 0.0001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.15\n early_stop_validations: -1\n episode_limit: 10000\n epoch_limit: 10\n\n# Validation parameters:\nvalidation:\n task:\n type: SimpleMolecules\n batch_size: *b\n split: validation\n resize_image: [87, 87]\n dataloader:\n num_workers: 4\n\n# Testing parameters:\ntest:\n task:\n type: SimpleMolecules\n batch_size: *b\n split: test\n resize_image: [87, 87]\n dataloader:\n num_workers: 4\n\ntest_mirror_blur:\n task:\n type: SimpleMolecules\n batch_size: *b\n split: test_mirror_blur\n resize_image: [87, 87]\n dataloader:\n num_workers: 4\n\ntest_on_grid:\n task:\n type: SimpleMolecules\n batch_size: *b\n split: test_on_grid\n resize_image: [87, 87]\n dataloader:\n num_workers: 4\n\ntest_handwritten:\n task:\n type: SimpleMolecules\n batch_size: *b\n split: test_handwritten\n resize_image: [87, 87]\n dataloader:\n num_workers: 4\n\n\n# Default components shared between all molecule classification pipelines.\npipeline:\n #disable: image_viewer\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 4\n streams:\n targets: targets\n predictions: predictions\n\n accuracy:\n priority: 5\n type: AccuracyStatistics\n streams:\n targets: targets\n predictions: predictions\n\n answer_decoder:\n priority: 6\n type: WordDecoder\n import_word_mappings_from_globals: True\n globals:\n word_mappings: label_word_mappings\n streams:\n inputs: predictions\n outputs: predicted_answers\n\n stream_viewer:\n priority: 7\n type: StreamViewer\n input_streams: labels, targets, predictions, predicted_answers\n\n image_viewer:\n priority: 100.5\n type: ImageViewer\n streams:\n images: images\n labels: labels\n answers: predicted_answers\n\n" }, { "path": "configs/molecule_classification/molecule_classification_convnet_softmax.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: molecule_classification/default_molecule_classification.yml\n\n# Pipeline - only the model-related components.\npipeline:\n # Model 1: 3 CNN layers.\n image_encoder:\n type: ConvNetEncoder\n priority: 1\n # Using default stream names, so the following could be removed (leaving it just for the clarity though).\n streams:\n inputs: images\n feature_maps: feature_maps\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n # TODO: change!\n #input_dims: [-1, 16, 107, 107]\n #output_dims: [-1, 183184]\n input_dims: [-1, 16, 9, 9]\n output_dims: [-1, 1296]\n priority: 2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model 2: 1 Fully connected layer with softmax acitvation.\n classifier:\n type: FeedForwardNetwork \n priority: 3\n streams:\n inputs: reshaped_maps\n # Using default stream name, so the following could be removed (leaving it just for the clarity though).\n predictions: predictions\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/molecule_classification/molecule_classification_vgg16_molecules.yml", "content": "# Load config defining MNIST tasks for training, validation and testing.\ndefault_configs: molecule_classification/default_molecule_classification.yml\n\n# Overrride some training parameters:\ntraining:\n task: \n resize_image: [224, 224]\n image_depth: 3\n\n# Override some validation parameters:\nvalidation:\n task:\n resize_image: [224, 224]\n image_depth: 3\n\n# Override some test parameters:\ntest:\n task:\n resize_image: [224, 224]\n image_depth: 3\n\ntest_mirror_blur:\n task:\n resize_image: [224, 224]\n image_depth: 3\n\ntest_on_grid:\n task:\n resize_image: [224, 224]\n image_depth: 3\n\ntest_handwritten:\n task:\n resize_image: [224, 224]\n image_depth: 3\n\n\n# Pipeline - only the model-related components.\npipeline:\n\n # Model 1: VGG\n image_encoder:\n type: GenericImageEncoder\n pretrained: False\n priority: 1\n # Using default stream names, so the following could be removed (leaving it just for the clarity though).\n streams:\n inputs: images\n outputs: vgg_images\n globals:\n output_size: num_classes\n\n\n # Model 2: 1 Fully connected layer with softmax acitvation.\n classifier:\n type: FeedForwardNetwork \n priority: 3\n streams:\n inputs: vgg_images\n # Using default stream name, so the following could be removed (leaving it just for the clarity though).\n predictions: predictions\n globals:\n input_size: num_classes\n prediction_size: num_classes\n\n#: pipeline\n" }, { "path": "configs/translation/eng_fra_translation_enc_attndec.yml", "content": "# This pipeline applied an encoder-decoder GRU with attention on the open Tatoeba translation sentence pairs. \n# Inspired by https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html .\n# Note that training will be slower than in the tutorial, as teacher forcing is not implemented here.\n\n# Training parameters:\ntraining:\n task:\n type: &p_type TranslationPairs\n data_folder: &data_folder ~/data/language_modeling/translation_pairs\n dataset: &dataset eng-fra\n subset: train\n sentence_length: 10\n batch_size: 64\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 1.0e-3\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 100\n task:\n type: *p_type\n data_folder: *data_folder\n dataset: *dataset\n subset: valid\n sentence_length: 10\n batch_size: 64\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n dataset: *dataset\n subset: test\n sentence_length: 10\n batch_size: 64\n\npipeline:\n\n # Source encoding - model 1.\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings: glove.6B.50d.txt\n data_folder: *data_folder\n source_vocabulary_files: eng-fra/eng.train.txt,eng-fra/eng.valid.txt,eng-fra/eng.test.txt\n vocabulary_mappings_file: eng-fra/eng.all.tokenized_words\n regenerate: True\n additional_tokens: ,\n import_word_mappings_from_globals: False\n export_word_mappings_to_globals: False\n fixed_padding: 10\n streams:\n inputs: sources\n outputs: embedded_sources\n \n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 2.1\n data_folder: *data_folder\n source_vocabulary_files: eng-fra/fra.train.txt,eng-fra/fra.valid.txt,eng-fra/fra.test.txt\n import_word_mappings_from_globals: False\n export_word_mappings_to_globals: True\n export_pad_mapping_to_globals: True\n eos_token: True\n fixed_padding: 10\n additional_tokens: ,\n regenerate: True\n streams:\n inputs: targets\n outputs: indexed_targets\n pad_index: tgt_pad_index\n \n # Single layer GRU Encoder\n encoder:\n type: RecurrentNeuralNetwork\n cell_type: GRU\n priority: 3\n initial_state: Trainable\n hidden_size: 50\n num_layers: 1\n use_logsoftmax: False\n output_last_state: True\n prediction_mode: Dense\n ffn_output: False\n streams:\n inputs: embedded_sources\n predictions: s2s_encoder_output\n output_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # Single layer GRU Decoder with attention\n decoder:\n type: AttentionDecoder\n priority: 4\n hidden_size: 50\n use_logsoftmax: False\n autoregression_length: 10\n prediction_mode: Dense\n streams:\n inputs: s2s_encoder_output\n predictions: s2s_decoder_output\n input_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # FF, to resize the from the output size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n streams:\n inputs: s2s_decoder_output\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_targets\n loss: loss\n globals:\n ignore_index: tgt_pad_index\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n ignored_words: [\"\", \"\"]\n streams:\n targets: indexed_targets\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: sources,targets,indexed_targets,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/tutorials/mnist_classification_convnet_softmax.yml", "content": "# Training parameters:\ntraining:\n task: \n type: MNIST\n batch_size: &b 64\n use_train_data: True\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [0, 55000]\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 0.15\n early_stop_validations: -1\n episode_limit: 10000\n epoch_limit: 10\n\n# Validation parameters:\nvalidation:\n task:\n type: MNIST\n batch_size: *b\n use_train_data: True # True because we are splitting the training set to: validation and training\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [55000, 60000]\n\n# Testing parameters:\ntest:\n task:\n type: MNIST\n batch_size: *b\n use_train_data: False # Test set.\n\npipeline:\n # Model 1: 3 CNN layers.\n image_encoder:\n type: ConvNetEncoder\n priority: 1\n # Using default stream names, so the following could be removed (leaving it just for the clarity though).\n streams:\n inputs: inputs\n feature_maps: feature_maps\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 1, 1]\n output_dims: [-1, 16]\n priority: 2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model 2: 1 Fully connected layer with softmax acitvation.\n classifier:\n type: FeedForwardNetwork \n priority: 3\n streams:\n inputs: reshaped_maps\n # Using default stream name, so the following could be removed (leaving it just for the clarity though).\n predictions: predictions\n globals:\n input_size: reshaped_maps_size\n prediction_size: num_classes\n\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 4\n # Using default stream names, so the following could be removed (leaving it just for the clarity though).\n streams:\n targets: targets\n predictions: predictions\n\n accuracy:\n priority: 5\n type: AccuracyStatistics\n # Using default stream names, so the following could be removed (leaving it just for the clarity though).\n streams:\n targets: targets\n predictions: predictions\n\n answer_decoder:\n priority: 6\n type: WordDecoder\n import_word_mappings_from_globals: True\n globals:\n word_mappings: label_word_mappings\n streams:\n inputs: predictions\n outputs: predicted_answers\n\n stream_viewer:\n priority: 7\n type: StreamViewer\n input_streams: labels, targets, predictions, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_all_bow_vgg16_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [2, 10, 100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: question_vocabulary_size\n\n question_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_questions\n outputs: question_activations\n globals:\n bow_size: question_vocabulary_size\n\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 3.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n \n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,99],[-1,10],[-1,100]]\n output_dims: [-1,209]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c1\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_all_rnn_vgg16_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_embeddings_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [100, 2, 10, 100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 1.3\n use_logsoftmax: False\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_embeddings_output_size\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 3.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n \n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10],[-1,100]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c1\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_image_cnn_softmax.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n\n # Image encoder.\n image_encoder:\n type: ConvNetEncoder\n priority: 1.1\n streams:\n inputs: images\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 26, 26]\n output_dims: [-1, 10816]\n priority: 1.2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model - softmax classifier.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [1000]\n priority: 3\n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: vocabulary_size_c1\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_image_size_softmax.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 1\n # Add input_size to globals, so classifier will use it.\n keys: input_size\n values: 2\n\n # Model - image size classifier.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 3\n streams:\n inputs: image_sizes\n globals:\n prediction_size: vocabulary_size_c1\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_question_mimic_rnn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n \n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 300\n pretrained_embeddings_file: mimic.fastText.no_clean.300d.pickled\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 3\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size_c1\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_question_onehot_bow.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: question_vocabulary_size\n\n question_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_questions\n outputs: bow_questions\n globals:\n bow_size: question_vocabulary_size\n\n # Model\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100, 100]\n priority: 3\n streams:\n inputs: bow_questions\n globals:\n input_size: question_vocabulary_size\n prediction_size: vocabulary_size_c1\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_question_rnn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n \n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 3\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size_c1\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/c1_classification_vf_question_rnn_separate_q_categorization.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c1_classification/default_c1_classification.yml\n\npipeline:\n \n ################# SHARED #################\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 0.2\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n ################# FLOW 0: CATEGORY #################\n\n # Model 2: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE\n load: ~/Documents/image-clef-2019/experiments/q_categorization/20190416_103111/checkpoints/vqa_med_question_categorization_rnn_best.pt\n freeze: True\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE\n load: ~/Documents/image-clef-2019/experiments/q_categorization/20190416_103111/checkpoints/vqa_med_question_categorization_rnn_best.pt\n freeze: True\n prediction_mode: Last\n priority: 0.4\n initial_state: Zero\n streams:\n inputs: embedded_questions\n predictions: predicted_question_categories\n globals:\n input_size: embeddings_size\n prediction_size: num_categories # C1,C2,C3,C4,BIN,UNK\n\n\n category_decoder:\n type: WordDecoder\n priority: 0.5\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predicted_question_categories\n outputs: predicted_question_categories_ids\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n ################# FLOW C1: question #################\n\n flow_c1_string_to_mask:\n type: StringToMask\n priority: 1.1\n globals:\n word_mappings: category_word_mappings\n streams:\n strings: predicted_question_categories_ids\n string_indices: flow_c1_targets\n masks: flow_c1_masks\n\n # Model 1: Embeddings\n flow_c1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: flow_c1_embedded_questions\n globals:\n embeddings_size: flow_c1_embeddings_size \n \n # Model 2: RNN\n flow_c1_lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 1.3\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: flow_c1_embedded_questions\n globals:\n input_size: flow_c1_embeddings_size\n prediction_size: vocabulary_size_c1\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c1_classification/default_c1_classification.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1\n export_sample_weights: ~/data/vqa-med/answers.c1.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c1.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1\n\n\npipeline:\n\n # Answer encoding.\n answer_indexer:\n type: LabelIndexer\n priority: 0.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c1\n word_mappings: word_mappings_c1\n\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 10.1\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_answers\n globals:\n vocabulary_size: vocabulary_size_c1\n word_mappings: word_mappings_c1\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.2\n targets_dim: 1\n streams:\n targets: answers_ids\n loss: loss\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.1\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.2\n # streams:\n # targets: answers_ids\n\n precision_recall:\n type: PrecisionRecallStatistics\n priority: 100.3\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n streams:\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c1\n num_classes: vocabulary_size_c1\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: questions,category_names,answers,predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet152_ewm_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n image_preprocessing: normalize\n # none | random_affine | random_horizontal_flip | normalize | all\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 48\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet152\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet152_rn_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 32\n # Appy all preprocessing/data augmentations.\n image_preprocessing: normalize\n # none | random_affine | random_horizontal_flip | normalize | all\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 32\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size,rn_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet152\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: Fusion: Relational Network #################\n # Object-object relations.\n question_image_fusion:\n priority: 4.1\n type: RelationalNetwork\n dropout_rate: 0.5\n g_theta_sizes: [512, 256]\n streams:\n question_encodings: question_activations\n outputs: fused_image_question_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_image_question_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [128,100]\n dropout_rate: 0.5\n streams:\n inputs: fused_image_question_activations\n predictions: rn_activation\n globals:\n input_size: fused_image_question_activation_size\n prediction_size: rn_activation_size\n\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 6th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [rn_activation,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet50_attn_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, attention_activation_size, question_image_activation_size]\n values: [100, 4196, 300]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n\n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet50\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: image-question fusion #################\n # Attention + FF.\n question_image_fusion:\n priority: 4.1\n type: QuestionDrivenAttention\n dropout_rate: 0.5\n latent_size: 100\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_activations\n outputs: attention_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: attention_activation_size\n\n classifier:\n priority: 5.1\n type: FeedForwardNetwork\n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: attention_activations\n globals:\n input_size: attention_activation_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet50_coattn_mfb_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, question_attention_activation_size, image_attention_activation_size, pooling_activation_size]\n values: [200, 800, 4096, 512]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n\n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: GRU\n prediction_mode: Dense\n use_logsoftmax: False\n output_last_state: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 128\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n # Self Attention for question.\n question_attention:\n priority: 1.4\n type: SelfAttention\n latent_size: 128\n num_attention_heads: 4\n streams:\n question_encodings: question_activations\n outputs: question_attention_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: question_attention_activation_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model_type: resnet50\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n\n image_attention:\n priority: 2.2\n type: QuestionDrivenAttention\n dropout_rate: 0.3\n latent_size: 1024\n output_mode: 'Image'\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_attention_activations\n outputs: image_attention_activations\n globals:\n question_encoding_size: question_attention_activation_size\n output_size: image_attention_activation_size\n\n ################# PIPE 3: image-question fusion #################\n # MFB\n question_image_fusion:\n priority: 3.1\n type: FactorizedBilinearPooling\n dropout_rate: 0.3\n latent_size: 512\n pool_factor: 2\n streams:\n image_encodings: image_attention_activations\n question_encodings: question_attention_activations\n outputs: pooling_activations\n globals:\n image_encoding_size: image_attention_activation_size\n question_encoding_size: question_attention_activation_size\n output_size: pooling_activation_size\n\n classifier:\n priority: 4.1\n type: FeedForwardNetwork\n hidden_sizes: [100]\n dropout_rate: 0.2\n streams:\n inputs: pooling_activations\n globals:\n input_size: pooling_activation_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet50_ewm_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet50\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet50_mfb_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, pooling_activation_size]\n values: [100, 2048, 256]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n\n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet50\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: image-question fusion #################\n # Attention + FF.\n question_image_fusion:\n priority: 4.1\n type: FactorizedBilinearPooling\n dropout_rate: 0.5\n latent_size: 256\n pool_factor: 2\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: pooling_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: pooling_activation_size # same as latent size\n\n classifier:\n priority: 5.1\n type: FeedForwardNetwork\n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: pooling_activations\n globals:\n input_size: pooling_activation_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_resnet50_rn_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 32\n # Appy all preprocessing/data augmentations.\n image_preprocessing: normalize\n # none | random_affine | random_horizontal_flip | normalize | all\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 32\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size,rn_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n model_type: resnet50\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: Fusion: Relational Network #################\n # Object-object relations.\n question_image_fusion:\n priority: 4.1\n type: RelationalNetwork\n dropout_rate: 0.5\n g_theta_sizes: [512, 256]\n streams:\n question_encodings: question_activations\n outputs: fused_image_question_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_image_question_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [128,100]\n dropout_rate: 0.5\n streams:\n inputs: fused_image_question_activations\n predictions: rn_activation\n globals:\n input_size: fused_image_question_activation_size\n prediction_size: rn_activation_size\n\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 6th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [rn_activation,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_selfattn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n batch_size: 48\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams:\n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, attention_activation_size]\n values: [100, 400]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n\n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Dense\n use_logsoftmax: False\n output_last_state: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 3: question attention #################\n # Self Attention for question.\n question_attention:\n priority: 4.1\n type: SelfAttention\n latent_size: 128\n num_attention_heads: 4\n streams:\n question_encodings: question_activations\n outputs: question_attention_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: attention_activation_size\n\n classifier:\n priority: 5.1\n type: FeedForwardNetwork\n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: question_attention_activations\n globals:\n input_size: attention_activation_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_vgg16_rn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n # Appy all preprocessing/data augmentations.\n image_preprocessing: normalize\n # none | random_affine | random_horizontal_flip | normalize | all\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size]\n values: [100]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Zero\n #dropout_rate: 0.5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: fusion + classification #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: RelationalNetwork\n dropout_rate: 0.5\n g_theta_sizes: [100, 100]\n streams:\n question_encodings: question_activations\n outputs: fused_image_question_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_image_question_activation_size\n\n classifier:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100,100]\n dropout_rate: 0.5\n streams:\n inputs: fused_image_question_activations\n globals:\n input_size: fused_image_question_activation_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_class_lstm_vgg16_rn_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\ntraining:\n task:\n # Appy all preprocessing/data augmentations.\n image_preprocessing: normalize\n # none | random_affine | random_horizontal_flip | normalize | all\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\nvalidation:\n task:\n question_preprocessing: lowercase,remove_punctuation,tokenize\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n streams: \n # Task is returning tokenized questions.\n questions: tokenized_questions\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size,rn_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: Fusion: Relational Network #################\n # Object-object relations.\n question_image_fusion:\n priority: 4.1\n type: RelationalNetwork\n dropout_rate: 0.5\n g_theta_sizes: [100, 100]\n streams:\n question_encodings: question_activations\n outputs: fused_image_question_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_image_question_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100,100]\n dropout_rate: 0.5\n streams:\n inputs: fused_image_question_activations\n predictions: rn_activation\n globals:\n input_size: fused_image_question_activation_size\n prediction_size: rn_activation_size\n\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 6th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [rn_activation,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_classification_all_rnn_vgg16_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_embeddings_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [100, 2, 10, 100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_embeddings_output_size\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n \n # 4th subpipeline: concatenation + FF.\n concat:\n priority: 4.1\n type: ConcatenateTensor\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10],[-1,100]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: output_size\n\n\n classifier:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_classification_all_rnn_vgg16_ewm.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size]\n values: [100, 100, 100]\n\n ################# PIPE 0: question #################\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Zero\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: fusion + classification #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n classifier:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: element_wise_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_classification_all_rnn_vgg16_ewm_size.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Trainable\n dropout_rate: 0.1\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 5: image-question-image size fusion + classification #################\n # Model - image size FFN.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c2\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_classification_all_rnn_vgg16_mcb.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, fused_image_question_activation_size]\n values: [200, 500, 100]\n\n ################# PIPE 0: question #################\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Zero\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: fusion + classification #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: CompactBilinearPooling\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: fused_image_question_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: fused_image_question_activation_size\n\n classifier:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: fused_image_question_activations\n globals:\n input_size: fused_image_question_activation_size\n prediction_size: vocabulary_size_c2\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/c2_word_answer_onehot_bow.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c2_classification/default_c2_classification.yml\n\n# Training parameters:\ntraining:\n task:\n batch_size: 128\n terminal_conditions:\n episode_limit: 1000\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: 128\n\npipeline:\n\n # Answer encoding.\n answer_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n preprocessing: lowercase,remove_punctuation\n remove_characters: [“,”,’]\n streams: \n inputs: answers\n outputs: tokenized_answer_words\n\n answer_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c2.preprocessed.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_answer_words\n outputs: encoded_answer_words\n globals:\n vocabulary_size: answer_words_vocabulary_size\n word_mappings: answer_words_word_mappings\n\n answer_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_answer_words\n outputs: bow_answer_words\n globals:\n bow_size: answer_words_vocabulary_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [50]\n dropout_rate: 0.5\n priority: 3\n streams:\n inputs: bow_answer_words\n globals:\n input_size: answer_words_vocabulary_size\n prediction_size: vocabulary_size_c2\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: answers, tokenized_answer_words, predicted_answers\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c2_classification/default_c2_classification.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C2\n export_sample_weights: ~/data/vqa-med/answers.c2.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c2.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C2\n\n\npipeline:\n\n # Answer encoding.\n answer_indexer:\n type: LabelIndexer\n priority: 0.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c2.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c2\n word_mappings: word_mappings_c2\n\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 10.1\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_answers\n globals:\n vocabulary_size: vocabulary_size_c2\n word_mappings: word_mappings_c2\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.2\n targets_dim: 1\n streams:\n targets: answers_ids\n loss: loss\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.1\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.2\n # streams:\n # targets: answers_ids\n\n precision_recall:\n type: PrecisionRecallStatistics\n priority: 100.3\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n streams:\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c2\n num_classes: vocabulary_size_c2\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: tokenized_questions,category_names,answers,predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_all_bow_vgg16_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [2, 10, 100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: question_vocabulary_size\n\n question_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_questions\n outputs: question_activations\n globals:\n bow_size: question_vocabulary_size\n\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 3.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n \n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,99],[-1,10],[-1,100]]\n output_dims: [-1,209]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c3\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_all_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [image_size_dims,latent_size]\n values: [2,100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 1.3\n use_logsoftmax: False\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: latent_size\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_ffn:\n type: FeedForwardNetwork \n #hidden_sizes: [100]\n priority: 2\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_dims\n prediction_size: latent_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: ConvNetEncoder\n priority: 3.1\n streams:\n inputs: images\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 26, 26]\n output_dims: [-1, 10816]\n priority: 3.2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model - feed-forward.\n image_ffn:\n type: FeedForwardNetwork \n hidden_sizes: [1000]\n priority: 3.3\n streams:\n inputs: reshaped_maps\n predictions: image_activations\n globals:\n input_size: reshaped_maps_size\n prediction_size: latent_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,100]]\n output_dims: [-1,300]\n streams:\n outputs: concatenated_activations\n globals:\n #input_sizes: [latent_size,latent_size,latent_size]\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c3\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_all_rnn_vgg16_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_embeddings_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [100, 2, 10, 100]\n\n # First subpipeline: question.\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 1.3\n use_logsoftmax: False\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_embeddings_output_size\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 2.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 3.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n \n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [question_activations,image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10],[-1,100]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c3\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_image_cnn_softmax.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n # Image encoder.\n image_encoder:\n type: ConvNetEncoder\n priority: 1.1\n streams:\n inputs: images\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 26, 26]\n output_dims: [-1, 10816]\n #input_dims: [-1, 16, 6, 6]\n #output_dims: [-1, 576]\n priority: 1.2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model - softmax classifier.\n classifier:\n type: FeedForwardNetwork \n #hidden_sizes: [32]\n priority: 3\n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: vocabulary_size_c3\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_image_plus_size_concat.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [image_size_dims,image_latent_size,image_size_latent_size]\n values: [2,100,10]\n\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_ffn:\n type: FeedForwardNetwork \n #hidden_sizes: [100]\n priority: 2\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_dims\n prediction_size: image_latent_size\n\n # 3rd subpipeline: image.\n # Image encoder.\n image_encoder:\n type: ConvNetEncoder\n priority: 3.1\n streams:\n inputs: images\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, 16, 26, 26]\n output_dims: [-1, 10816]\n priority: 3.2\n streams:\n inputs: feature_maps\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model - feed-forward.\n image_ffn:\n type: FeedForwardNetwork \n hidden_sizes: [1000]\n priority: 3.3\n streams:\n inputs: reshaped_maps\n predictions: image_activations\n globals:\n input_size: reshaped_maps_size\n prediction_size: image_size_latent_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [image_size_activations,image_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n #input_sizes: [latent_size,latent_size,latent_size]\n output_size: output_size\n\n\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [50]\n priority: 4.2\n streams:\n inputs: concatenated_activations\n globals:\n input_size: output_size\n prediction_size: vocabulary_size_c3\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_image_size_softmax.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n global_publisher:\n type: GlobalVariablePublisher\n priority: 1\n # Add input_size to globals, so classifier will use it.\n keys: input_size\n values: 2\n\n # Model - image size classifier.\n classifier:\n type: FeedForwardNetwork \n #hidden_sizes: [100]\n priority: 3\n streams:\n inputs: image_sizes\n globals:\n prediction_size: vocabulary_size_c3\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_image_softmax.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\ntraining:\n task:\n resize_image: &resize_image [28, 28]\n\nvalidation:\n task:\n resize_image: *resize_image\n\n\npipeline:\n\n # Reshape inputs\n reshaper:\n type: ReshapeTensor\n priority: 1.1\n input_dims: [-1, 3, 28, 28]\n output_dims: [-1, 2352]\n streams:\n inputs: images\n outputs: reshaped_maps\n globals:\n output_size: reshaped_maps_size\n\n # Model - softmax classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 1.2\n hidden: [100]\n streams:\n inputs: reshaped_maps\n globals:\n input_size: reshaped_maps_size\n prediction_size: vocabulary_size_c3\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_image_vgg16_softmax.yml", "content": " # Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 1.1\n streams:\n inputs: images\n outputs: vgg_images\n globals:\n output_size: vocabulary_size_c3\n\n # Model - softmax classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 3\n streams:\n inputs: vgg_images\n globals:\n input_size: vocabulary_size_c3\n prediction_size: vocabulary_size_c3\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_question_onehot_bow.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: sentence_vocabulary_size\n\n question_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_questions\n outputs: bow_questions\n globals:\n bow_size: sentence_vocabulary_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [100, 100]\n priority: 3\n streams:\n inputs: bow_questions\n globals:\n input_size: sentence_vocabulary_size\n prediction_size: vocabulary_size_c3\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/c3_classification_question_rnn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c3_classification/default_c3_classification.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 3\n initial_state: Zero\n #num_layers: 5\n hidden_size: 50\n streams:\n inputs: embedded_questions\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size_c3\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c3_classification/default_c3_classification.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C3\n export_sample_weights: ~/data/vqa-med/answers.c3.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c3.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C3\n\n\npipeline:\n\n # Answer encoding.\n answer_indexer:\n type: LabelIndexer\n priority: 0.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c3.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c3\n word_mappings: word_mappings_c3\n\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 10.1\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_answers\n globals:\n vocabulary_size: vocabulary_size_c3\n word_mappings: word_mappings_c3\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.2\n targets_dim: 1\n streams:\n targets: answers_ids\n loss: loss\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.1\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.2\n # streams:\n # targets: answers_ids\n\n precision_recall:\n type: PrecisionRecallStatistics\n priority: 100.3\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n streams:\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c3\n num_classes: vocabulary_size_c3\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: questions,category_names,answers,predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_classification_all_rnn_vgg16_ewm_size.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c4_classification/default_c4_classification.yml\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 100, 2, 10]\n\n ################# PIPE 0: question #################\n # Questions encoding.\n question_tokenizer:\n priority: 1.1\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: Embeddings\n question_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n question_lstm:\n priority: 1.3\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n use_logsoftmax: False\n initial_state: Zero\n hidden_size: 50\n #dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: image #################\n # Image encoder.\n image_encoder:\n priority: 3.1\n type: GenericImageEncoder\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 4: image-question-image size fusion + classification #################\n # 2nd subpipeline: image size.\n # Model - image size classifier.\n image_size_encoder:\n priority: 5.1\n type: FeedForwardNetwork \n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n # 4th subpipeline: concatenation + FF.\n concat:\n priority: 5.2\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatentated_activations_size\n\n\n classifier:\n priority: 5.3\n type: FeedForwardNetwork \n hidden_sizes: [500]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n globals:\n input_size: concatentated_activations_size\n prediction_size: vocabulary_size_c4\n\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_enc_attndec.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n batch_size: 64\n categories: C4\n question_preprocessing: lowercase, remove_punctuation, tokenize, random_remove_stop_words #,random_shuffle_words \n answer_preprocessing: lowercase, remove_punctuation, tokenize\n export_sample_weights: ~/data/vqa-med/answers.c4.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c4.weights.csv\n dataloader:\n num_workers: 2\n # Termination.\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: 64\n categories: C4\n question_preprocessing: lowercase, remove_punctuation, tokenize\n answer_preprocessing: lowercase, remove_punctuation, tokenize\n dataloader:\n num_workers: 2\n\npipeline:\n\n # Question embeddings\n question_embeddings:\n priority: 1.0\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n fixed_padding: 10\n additional_tokens: \n streams:\n inputs: questions\n outputs: embedded_questions\n\n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 1.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n import_word_mappings_from_globals: False\n export_word_mappings_to_globals: True\n export_pad_mapping_to_globals: True\n eos_token: True\n fixed_padding: 10\n streams:\n inputs: answers\n outputs: indexed_answers\n globals:\n vocabulary_size: ans_vocabulary_size\n word_mappings: ans_word_mappings\n pad_index: ans_pad_index\n\n # Single layer GRU Encoder\n encoder:\n type: RecurrentNeuralNetwork\n cell_type: GRU\n priority: 3\n initial_state: Trainable\n hidden_size: 100\n num_layers: 1\n use_logsoftmax: False\n output_last_state: True\n prediction_mode: Dense\n ffn_output: False\n dropout_rate: 0.1\n streams:\n inputs: embedded_questions\n predictions: s2s_encoder_output\n output_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # Single layer GRU Decoder with attention\n decoder:\n type: AttentionDecoder\n priority: 4\n hidden_size: 100\n use_logsoftmax: False\n autoregression_length: 10\n prediction_mode: Dense\n dropout_rate: 0.1\n streams:\n inputs: s2s_encoder_output\n predictions: s2s_decoder_output\n input_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # FF, to resize the from the output size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n dropout_rate: 0.1\n streams:\n inputs: s2s_decoder_output\n globals:\n input_size: embeddings_size\n prediction_size: ans_vocabulary_size\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_answers\n loss: loss\n globals:\n ignore_index: ans_pad_index\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: ~/data/vqa-med\n import_word_mappings_from_globals: True\n globals:\n word_mappings: ans_word_mappings\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n globals:\n word_mappings: ans_word_mappings\n streams:\n targets: indexed_answers\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: questions,answers,indexed_answers,prediction_sentences\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_enc_attndec_resnet152_ewm_cat_is.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n batch_size: 200 # requires to use 4 GPUs!\n categories: C4\n question_preprocessing: lowercase, remove_punctuation, tokenize #, random_remove_stop_words #,random_shuffle_words \n answer_preprocessing: lowercase, remove_punctuation, tokenize\n export_sample_weights: ~/data/vqa-med/answers.c4.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c4.weights.csv\n dataloader:\n num_workers: 4\n # Termination.\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: 200\n categories: C4\n question_preprocessing: lowercase, remove_punctuation, tokenize\n answer_preprocessing: lowercase, remove_punctuation, tokenize\n dataloader:\n num_workers: 4\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_encoder_output_size, element_wise_activation_size,image_size_encoder_input_size, image_size_encoder_output_size]\n values: [100, 100, 100, 2, 10]\n\n # Question embeddings\n question_embeddings:\n priority: 1.0\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n fixed_padding: 10 # The longest question! max is 19!\n additional_tokens: ,\n streams:\n inputs: questions\n outputs: embedded_questions\n\n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 1.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n import_word_mappings_from_globals: False\n export_word_mappings_to_globals: True\n export_pad_mapping_to_globals: True\n additional_tokens: ,\n eos_token: True\n fixed_padding: 10 # The longest question! max is 19!\n streams:\n inputs: answers\n outputs: indexed_answers\n globals:\n vocabulary_size: ans_vocabulary_size\n word_mappings: ans_word_mappings\n pad_index: ans_pad_index\n\n # Image encoder.\n image_encoder:\n priority: 2.0\n type: GenericImageEncoder\n model_type: resnet152\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n # Single layer GRU Encoder\n encoder:\n priority: 3\n type: RecurrentNeuralNetwork\n cell_type: GRU\n initial_state: Trainable\n hidden_size: 100\n num_layers: 1\n use_logsoftmax: False\n output_last_state: True\n prediction_mode: Dense\n ffn_output: False\n dropout_rate: 0.1\n streams:\n inputs: embedded_questions\n predictions: s2s_encoder_output\n output_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size \n\n reshaper_1:\n priority: 3.01\n type: ReshapeTensor\n input_dims: [-1, 1, 100]\n output_dims: [-1, 100]\n streams:\n inputs: s2s_state_output\n outputs: s2s_state_output_reshaped\n globals:\n output_size: s2s_state_output_reshaped_size\n\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 3.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: s2s_state_output_reshaped\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 3.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n reshaper_2:\n priority: 3.3\n type: ReshapeTensor\n input_dims: [-1, 100]\n output_dims: [-1, 1, 100]\n streams:\n inputs: question_image_activations\n outputs: question_image_activations_reshaped\n globals:\n output_size: question_image_activations_reshaped_size\n\n # Single layer GRU Decoder with attention\n decoder:\n type: AttentionDecoder\n priority: 4\n hidden_size: 100\n use_logsoftmax: False\n autoregression_length: 10 # Current implementation requires this value to be equal to fixed_padding in SentenceEmbeddings/Indexer...\n prediction_mode: Dense\n dropout_rate: 0.1\n streams:\n inputs: s2s_encoder_output\n predictions: s2s_decoder_output\n input_state: question_image_activations_reshaped\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size \n\n # FF, to resize the from the output size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n dropout_rate: 0.1\n streams:\n inputs: s2s_decoder_output\n globals:\n input_size: element_wise_activation_size\n prediction_size: ans_vocabulary_size\n\n# Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_answers\n loss: loss\n globals:\n ignore_index: ans_pad_index\n\n # Prediction decoding.\n prediction_decoder:\n priority: 10\n type: SentenceIndexer\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: ~/data/vqa-med\n import_word_mappings_from_globals: True\n globals:\n word_mappings: ans_word_mappings\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n globals:\n word_mappings: ans_word_mappings\n streams:\n targets: indexed_answers\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: questions,answers,indexed_answers,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_frozen_if_gru_dec.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml,\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml,\n vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nc4_hyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n batch_size: &batch_size 256\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C4\n export_sample_weights: ~/data/vqa-med/answers.c4.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c4.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C4\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Answer encoding.\n pipe6_c123_binary_yn_answer_indexer:\n priority: 6.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n # Model 4: FFN C123 answering\n pipe6_c123_binary_yn_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_binary_yn_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_binary_yn_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_binary_yn_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_word_answer_glove_sum.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c4_classification/default_c4_classification.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n \n# Validation parameters:\nvalidation:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [answer_word_embeddings_size]\n values: [100]\n\n # Answer encoding.\n answer_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n preprocessing: lowercase,remove_punctuation\n remove_characters: [“,”,’]\n streams: \n inputs: answers\n outputs: tokenized_answer_words\n\n # Model 1: Embeddings\n answer_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 100\n pretrained_embeddings_file: glove.6B.100d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_answer_words\n outputs: encoded_answer_words\n globals:\n vocabulary_size: answer_words_vocabulary_size\n word_mappings: answer_words_word_mappings\n\n answer_reduction:\n type: ReduceTensor\n priority: 1.3\n num_inputs_dims: 3\n reduction_dim: 1\n reduction_type: sum\n keepdim: False\n streams:\n inputs: encoded_answer_words\n outputs: reduced_answers\n globals:\n input_size: answer_word_embeddings_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [500, 500]\n dropout_rate: 0.5\n priority: 3\n streams:\n inputs: reduced_answers\n globals:\n input_size: answer_word_embeddings_size\n prediction_size: vocabulary_size_c4\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: answers, tokenized_answer_words, predicted_answers\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_word_answer_mimic_sum.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c4_classification/default_c4_classification.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n \n# Validation parameters:\nvalidation:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n\n\npipeline:\n\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [answer_word_embeddings_size]\n values: [300]\n\n # Answer encoding.\n answer_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n preprocessing: lowercase,remove_punctuation\n remove_characters: [“,”,’]\n streams: \n inputs: answers\n outputs: tokenized_answer_words\n\n # Model 1: Embeddings\n answer_embeddings:\n priority: 1.2\n type: SentenceEmbeddings\n embeddings_size: 300\n pretrained_embeddings_file: mimic.fastText.no_clean.300d.pickled\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_answer_words\n outputs: encoded_answer_words\n globals:\n vocabulary_size: answer_words_vocabulary_size\n word_mappings: answer_words_word_mappings\n\n answer_reduction:\n type: ReduceTensor\n priority: 1.3\n num_inputs_dims: 3\n reduction_dim: 1\n reduction_type: sum\n keepdim: False\n streams:\n inputs: encoded_answer_words\n outputs: reduced_answers\n globals:\n input_size: answer_word_embeddings_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [500, 500]\n dropout_rate: 0.5\n priority: 3\n streams:\n inputs: reduced_answers\n globals:\n input_size: answer_word_embeddings_size\n prediction_size: vocabulary_size_c4\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: answers, tokenized_answer_words, predicted_answers\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_word_answer_onehot_bow.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c4_classification/default_c4_classification.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n \n# Validation parameters:\nvalidation:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n\n\npipeline:\n # Answer encoding.\n answer_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n preprocessing: lowercase,remove_punctuation\n remove_characters: [“,”,’]\n streams: \n inputs: answers\n outputs: tokenized_answer_words\n\n answer_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_answer_words\n outputs: encoded_answer_words\n globals:\n vocabulary_size: answer_words_vocabulary_size\n word_mappings: answer_words_word_mappings\n\n answer_bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_answer_words\n outputs: bow_answer_words\n globals:\n bow_size: answer_words_vocabulary_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [500, 500]\n dropout_rate: 0.5\n priority: 3\n streams:\n inputs: bow_answer_words\n globals:\n input_size: answer_words_vocabulary_size\n prediction_size: vocabulary_size_c4\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: answers, tokenized_answer_words, predicted_answers\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/c4_word_answer_onehot_sum.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/c4_classification/default_c4_classification.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n \n# Validation parameters:\nvalidation:\n task:\n categories: C4\n batch_size: 512\n # In here we won't use images at all.\n stream_images: False\n dataloader:\n num_workers: 0\n\n\npipeline:\n # Answer encoding.\n answer_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n preprocessing: lowercase,remove_punctuation\n remove_characters: [“,”,’]\n streams: \n inputs: answers\n outputs: tokenized_answer_words\n\n answer_onehot_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_answer_words\n outputs: encoded_answer_words\n globals:\n vocabulary_size: answer_words_vocabulary_size\n word_mappings: answer_words_word_mappings\n\n answer_to_tensor:\n type: ListToTensor\n priority: 1.3\n num_inputs_dims: 3\n streams:\n inputs: encoded_answer_words\n outputs: tensor_answer_words\n globals:\n input_size: answer_words_vocabulary_size\n\n\n answer_reduction:\n type: ReduceTensor\n priority: 1.4\n num_inputs_dims: 3\n reduction_dim: 1\n reduction_type: sum\n keepdim: False\n streams:\n inputs: tensor_answer_words\n outputs: reduced_answer_words\n globals:\n input_size: answer_words_vocabulary_size\n\n # Model.\n classifier:\n type: FeedForwardNetwork \n hidden_sizes: [500, 500]\n dropout_rate: 0.5\n priority: 3\n streams:\n inputs: reduced_answer_words\n globals:\n input_size: answer_words_vocabulary_size\n prediction_size: vocabulary_size_c4\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: answers, tokenized_answer_words, predicted_answers\n \n#: pipeline\n" }, { "path": "configs/vqa_med_2019/c4_classification/default_c4_classification.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C4\n export_sample_weights: ~/data/vqa-med/answers.c4.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c4.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C4\n\n\npipeline:\n\n # Answer encoding.\n answer_indexer:\n type: LabelIndexer\n priority: 0.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c4.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c4\n word_mappings: word_mappings_c4\n\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 10.1\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_answers\n globals:\n vocabulary_size: vocabulary_size_c4\n word_mappings: word_mappings_c4\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.2\n targets_dim: 1\n streams:\n targets: answers_ids\n loss: loss\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.1\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.2\n # streams:\n # targets: answers_ids\n\n precision_recall:\n type: PrecisionRecallStatistics\n priority: 100.3\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c4\n num_classes: vocabulary_size_c4\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: questions,category_names,answers,predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/default_vqa_med_2019.yml", "content": "# Training parameters:\ntraining:\n task:\n type: &p_type VQAMED2019\n data_folder: &data_folder ~/data/vqa-med\n split: training_validation\n resize_image: &resize_image [224, 224]\n batch_size: 64\n # Generate and export shuffled indices.\n shuffle_indices: True\n export_indices: shuffled_indices.npy\n\n # Default sampler during training.\n sampler:\n type: kFoldWeightedRandomSampler\n folds: 20\n epochs_per_fold: 1000 # NEVER CHANGE FOLD!\n # Use four workers for loading images.\n dataloader:\n num_workers: 4\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 10\n task:\n type: *p_type\n data_folder: *data_folder\n split: training_validation\n resize_image: *resize_image \n batch_size: 64\n # Import shuffled indices.\n import_indices: shuffled_indices.npy\n\n # Default sampler during validation.\n sampler:\n type: kFoldRandomSampler\n folds: 20\n epochs_per_fold: 1000 # NEVER CHANGE FOLD!\n # Use four workers for loading images.\n dataloader:\n num_workers: 4\n" }, { "path": "configs/vqa_med_2019/evaluation/deepta/glove_gru_resnet50_coattn_mfb_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model resnet50\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n # image_encoder_output_size_val: &image_encoder_output_size_val 2048\n image_attention_multihead_size_val: &image_attention_multihead_size_val 2\n # image_attention_output_size_val: &image_attention_output_size_val 4096\n\n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.100d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 100\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 128\n question_encoder_output_size_val: &question_encoder_output_size_val 200\n question_attention_multihead_size_val: &question_attention_multihead_size_val 4\n question_attention_output_size_val: &question_attention_output_size_val 800\n\n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type FactorizedBilinearPooling\n # Options: LowRankBilinearPooling | FactorizedBilinearPooling (component: question_image_fusion)\n question_image_fusion_size_val: &question_image_fusion_size_val 512\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 522\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [100]\n\n batch_size: &batch_size 64\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing\n streams:\n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 2000 #10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing\n streams:\n questions: tokenized_questions\n\n\npipeline:\n\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, question_attention_output_size, image_size_encoder_input_size, image_size_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, *question_attention_output_size_val, 2, *image_size_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size\n\n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: GRU\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Dense\n initial_state: Trainable\n use_logsoftmax: False\n output_last_state: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n\n # Model 3: self attention for question.\n question_attention:\n priority: 1.3\n type: SelfAttention\n latent_size: *question_encoder_lstm_size_val\n num_attention_heads: 4\n streams:\n question_encodings: question_activations\n outputs: question_attention_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: question_attention_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model_type: *image_encoder_model\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n\n image_attention:\n priority: 2.2\n type: QuestionDrivenAttention\n dropout_rate: 0.3\n latent_size: 256\n output_mode: 'Image'\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_attention_activations\n outputs: image_attention_activations\n globals:\n question_encoding_size: question_attention_output_size\n output_size: image_attention_output_size\n\n ################# PIPE 3: image-question fusion #################\n # MFB.\n question_image_fusion:\n priority: 3.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n latent_size: 512\n pool_factor: 2\n streams:\n image_encodings: image_attention_activations\n question_encodings: question_attention_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_attention_output_size\n question_encoding_size: question_attention_output_size\n # output_size: image_attention_output_size #fused_activation_size\n\n ################# PIPE 4: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 4.1\n type: FeedForwardNetwork\n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation\n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [fused_activations,image_size_activations]\n # ConcatenateTensor\n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 7: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 7.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, pipe0_predicted_question_categories_names,\n answers, predicted_answers\n" }, { "path": "configs/vqa_med_2019/evaluation/deepta/glove_gru_vgg16_coattn_mfb_is_cat_ffn_c1234_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model vgg16\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n # image_encoder_output_size_val: &image_encoder_output_size_val 2048\n image_attention_multihead_size_val: &image_attention_multihead_size_val 2\n # image_attention_output_size_val: &image_attention_output_size_val 4096\n\n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.100d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 100\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 128\n question_encoder_output_size_val: &question_encoder_output_size_val 200\n question_attention_multihead_size_val: &question_attention_multihead_size_val 4\n question_attention_output_size_val: &question_attention_output_size_val 800\n\n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type FactorizedBilinearPooling\n # Options: LowRankBilinearPooling | FactorizedBilinearPooling (component: question_image_fusion)\n question_image_fusion_size_val: &question_image_fusion_size_val 512\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 522\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [100]\n\n batch_size: &batch_size 64\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: all #C1,C2,C3 # TODO: all\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing\n streams:\n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 5000 #10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: all #C1,C2,C3 # TODO: all\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing\n streams:\n questions: tokenized_questions\n\n\npipeline:\n\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, question_attention_output_size, image_size_encoder_input_size, image_size_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, *question_attention_output_size_val, 2, *image_size_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.all.word.mappings.csv # TODO: all\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c1234_binary_yn # TODO: rename\n word_mappings: word_mappings_c1234_binary_yn\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size\n\n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: GRU\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Dense\n initial_state: Trainable\n use_logsoftmax: False\n output_last_state: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n\n # Model 3: self attention for question.\n question_attention:\n priority: 1.3\n type: SelfAttention\n latent_size: *question_encoder_lstm_size_val\n num_attention_heads: 4\n streams:\n question_encodings: question_activations\n outputs: question_attention_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: question_attention_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model_type: *image_encoder_model\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n\n image_attention:\n priority: 2.2\n type: QuestionDrivenAttention\n dropout_rate: 0.3\n latent_size: 256\n output_mode: 'Image'\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_attention_activations\n outputs: image_attention_activations\n globals:\n question_encoding_size: question_attention_output_size\n output_size: image_attention_output_size\n\n ################# PIPE 3: image-question fusion #################\n # MFB.\n question_image_fusion:\n priority: 3.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n latent_size: 512\n pool_factor: 2\n streams:\n image_encodings: image_attention_activations\n question_encodings: question_attention_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_attention_output_size\n question_encoding_size: question_attention_output_size\n # output_size: image_attention_output_size #fused_activation_size\n\n ################# PIPE 4: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 4.1\n type: FeedForwardNetwork\n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation\n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [fused_activations,image_size_activations]\n # ConcatenateTensor\n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c1234_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 7: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 7.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, pipe0_predicted_question_categories_names,\n answers, predicted_answers\n" }, { "path": "configs/vqa_med_2019/evaluation/example_mimic_lstm_vgg16_ewm_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model resnet50\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n image_encoder_output_size_val: &image_encoder_output_size_val 74\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings mimic.fastText.no_clean.300d.pickled\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 300\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 101\n question_encoder_output_size_val: &question_encoder_output_size_val 103\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type LowRankBilinearPooling\n # Options: LowRankBilinearPooling | ? (component: question_image_fusion)\n question_image_fusion_size_val: &question_image_fusion_size_val 115\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 11\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 126\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [83]\n\n # Parameters related to GPU/CPU distribution.\n batch_size: &batch_size 100\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val, *image_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/frozen_if_ffn_c1234_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml,\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml\n #vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nc123_hyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500,500]\n\n batch_size: &batch_size 256\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: all\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: all\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 6: C1 + C2 + C3 + C4 questions #################\n\n # Answer encoding.\n pipe6_c1234_answer_indexer:\n priority: 6.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.all.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c1234_binary_yn\n word_mappings: word_mappings_c1234_binary_yn\n\n\n # Model 4: FFN C123 answering\n pipe6_c1234_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c1234_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c1234_binary_yn\n\n pipe6_c1234_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c1234_predictions\n targets: answers_ids\n loss: pipe6_c1234_loss\n\n pipe6_c1234_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c1234_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c1234_binary_yn\n statistics:\n precision: pipe6_c1234_precision\n recall: pipe6_c1234_recall\n f1score: pipe6_c1234_f1score\n\n # C123 Predictions decoder.\n pipe6_c1234_binary_yn_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c1234_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c1234_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/frozen_if_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml,\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml,\n vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nc123_hyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n batch_size: &batch_size 256\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Answer encoding.\n pipe6_c123_binary_yn_answer_indexer:\n priority: 6.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n # Model 4: FFN C123 answering\n pipe6_c123_binary_yn_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_binary_yn_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_binary_yn_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_binary_yn_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml,\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml,\n vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nc123_hyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n batch_size: &batch_size 512\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: all\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Use workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: all\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 6: masks for all 5 classifiers #################\n\n # Add global variables.\n c1234_global_publisher:\n type: GlobalVariablePublisher\n priority: 6.0\n # Add input_size to globals.\n keys: [category_c1_word_to_ix, category_c2_word_to_ix, category_c3_word_to_ix, category_c4_word_to_ix, category_binary_word_to_ix]\n values: [{\"C1\": 0}, {\"C2\": 0}, {\"C3\": 0}, {\"C4\": 0}, {\"BINARY\": 0}]\n\n # Sample masking based on categories.\n c1_string_to_mask:\n priority: 6.1\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n # Input: predicted categories by the \"question categorizer\" pipeline.\n strings: predicted_category_names\n masks: c1_masks\n\n c2_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c2_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c2_masks\n\n c3_string_to_mask:\n priority: 6.3\n type: StringToMask\n globals:\n word_mappings: category_c3_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c3_masks\n\n c4_string_to_mask:\n priority: 6.4\n type: StringToMask\n globals:\n word_mappings: category_c4_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c4_masks\n\n binary_string_to_mask:\n priority: 6.5\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: predicted_category_names\n masks: binary_masks\n\n ################# PIPE 7: answers/targets for all 5 classifiers #################\n\n c1_answer_indexer:\n type: LabelIndexer\n priority: 7.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c1_without_yn_answers_ids\n globals:\n vocabulary_size: c1_without_yn_vocabulary_size\n word_mappings: c1_without_yn_word_mappings\n\n c2_answer_indexer:\n type: LabelIndexer\n priority: 7.2\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c2.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c2_answers_ids\n globals:\n vocabulary_size: c2_vocabulary_size\n word_mappings: c2_word_mappings\n\n c3_answer_indexer:\n type: LabelIndexer\n priority: 7.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c3.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c3_answers_ids\n globals:\n vocabulary_size: c3_vocabulary_size\n word_mappings: c3_word_mappings\n\n c4_answer_indexer:\n type: LabelIndexer\n priority: 7.4\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c4_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c4_without_yn_answers_ids\n globals:\n vocabulary_size: c4_without_yn_vocabulary_size\n word_mappings: c4_without_yn_word_mappings\n\n binary_answer_indexer:\n type: LabelIndexer\n priority: 7.6\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: binary_yn_answers_ids\n globals:\n vocabulary_size: binary_yn_vocabulary_size\n word_mappings: binary_yn_word_mappings\n\n # Answer encoding for all classes.\n c1234_answer_indexer:\n priority: 7.7\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.all.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: c1234_binary_yn_vocabulary_size\n word_mappings: c1234_binary_yn_word_mappings\n\n\n ################# PIPE 8: C1 questions #################\n\n # Model: FFN for C1 answering\n c1_answer_classifier:\n priority: 8.1\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: c1_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: c1_without_yn_vocabulary_size\n\n c1_nllloss:\n priority: 8.2\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c1_masks\n predictions: c1_predictions\n targets: c1_without_yn_answers_ids\n loss: c1_loss\n\n c1_precision_recall:\n priority: 8.3\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c1_masks\n predictions: c1_predictions\n targets: c1_without_yn_answers_ids\n globals:\n word_mappings: c1_without_yn_word_mappings\n statistics:\n precision: c1_precision\n recall: c1_recall\n f1score: c1_f1score\n\n c1_prediction_decoder:\n priority: 8.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c1_predictions\n outputs: c1_without_yn_prediction_labels\n globals:\n word_mappings: c1_without_yn_word_mappings\n\n\n ################# PIPE 9: C2 questions #################\n\n # Model: FFN for C2 answering\n c2_answer_classifier:\n priority: 9.1\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: c2_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: c2_vocabulary_size\n\n c2_nllloss:\n priority: 9.2\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c2_masks\n predictions: c2_predictions\n targets: c2_answers_ids\n loss: c2_loss\n\n c2_precision_recall:\n priority: 9.3\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c2_masks\n predictions: c2_predictions\n targets: c2_answers_ids\n globals:\n word_mappings: c2_word_mappings\n statistics:\n precision: c2_precision\n recall: c2_recall\n f1score: c2_f1score\n\n c2_prediction_decoder:\n priority: 9.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c2_predictions\n outputs: c2_prediction_labels\n globals:\n word_mappings: c2_word_mappings\n\n\n ################# PIPE 10: C3 questions #################\n\n # Model: FFN for c3 answering\n c3_answer_classifier:\n priority: 10.1\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: c3_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: c3_vocabulary_size\n\n c3_nllloss:\n priority: 10.2\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c3_masks\n predictions: c3_predictions\n targets: c3_answers_ids\n loss: c3_loss\n\n c3_precision_recall:\n priority: 10.3\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c3_masks\n predictions: c3_predictions\n targets: c3_answers_ids\n globals:\n word_mappings: c3_word_mappings\n statistics:\n precision: c3_precision\n recall: c3_recall\n f1score: c3_f1score\n\n c3_prediction_decoder:\n priority: 10.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c3_predictions\n outputs: c3_prediction_labels\n globals:\n word_mappings: c3_word_mappings\n\n\n ################# PIPE 11: C4 questions #################\n\n # Model: FFN for C4 answering\n c4_answer_classifier:\n priority: 11.1\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: c4_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: c4_without_yn_vocabulary_size\n\n c4_nllloss:\n priority: 11.2\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c4_masks\n predictions: c4_predictions\n targets: c4_without_yn_answers_ids\n loss: c4_loss\n\n c4_precision_recall:\n priority: 11.3\n type: PrecisionRecallStatistics\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c4_masks\n predictions: c4_predictions\n targets: c4_without_yn_answers_ids\n globals:\n word_mappings: c4_without_yn_word_mappings\n statistics:\n precision: c4_precision\n recall: c4_recall\n f1score: c4_f1score\n\n c4_prediction_decoder:\n priority: 11.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c4_predictions\n outputs: c4_without_yn_prediction_labels\n globals:\n word_mappings: c4_without_yn_word_mappings\n\n\n ################# PIPE 12: BINARY Y/N questions #################\n\n # Model: FFN for BINARY Y/N answering\n binary_answer_classifier:\n priority: 12.1\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: binary_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: binary_yn_vocabulary_size\n\n binary_nllloss:\n priority: 12.2\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: binary_masks\n predictions: binary_predictions\n targets: binary_yn_answers_ids\n loss: binary_loss\n\n binary_precision_recall:\n priority: 12.3\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: binary_masks\n predictions: binary_predictions\n targets: binary_yn_answers_ids\n globals:\n word_mappings: binary_yn_word_mappings\n statistics:\n precision: binary_precision\n recall: binary_recall\n f1score: binary_f1score\n\n binary_prediction_decoder:\n priority: 12.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: binary_predictions\n outputs: binary_prediction_labels\n globals:\n word_mappings: binary_yn_word_mappings\n\n ################# PIPE 13: MERGE ANSWERS #################\n\n # Merge predictions.\n pipe9_merged_predictions:\n type: JoinMaskedPredictions\n priority: 13.1\n # Names of used input streams.\n input_prediction_streams: [c1_predictions, c2_predictions, c3_predictions, c4_predictions, binary_predictions]\n input_mask_streams: [c1_masks, c2_masks, c3_masks, c4_masks, binary_masks]\n input_word_mappings: [c1_without_yn_word_mappings, c2_word_mappings, c3_word_mappings, c4_without_yn_word_mappings, binary_yn_word_mappings]\n globals: \n output_word_mappings: c1234_binary_yn_word_mappings\n streams:\n output_strings: predicted_answers\n output_indices: merged_prediction_indices\n\n # Statistics.\n pipe9_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 13.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n globals:\n word_mappings: c1234_binary_yn_word_mappings\n streams:\n targets: answers_ids\n predictions: merged_prediction_indices\n statistics:\n precision: merged_precision\n recall: merged_recall\n f1score: merged_f1score\n\n # Viewers.\n viewer:\n priority: 13.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n c1_masks, c2_masks, c3_masks, c4_masks, binary_masks,\n answers,\n c1_without_yn_prediction_labels,\n c2_prediction_labels,\n c3_prediction_labels,\n c4_without_yn_prediction_labels,\n binary_prediction_labels,\n predicted_answers\n\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_support_c1234yn_5losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml,\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml,\n vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nc1234_hyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Hidden size of C1/C2/C3 \"supporting fact\" networks.\n c123_support_hidden_sizes_val: &c123_support_hidden_sizes_val [500]\n\n # Size of output of C1/C2/C3 \"supporting fact\" networks.\n c123_support_output_size_val: &c123_support_output_size_val 100\n\n # Hidden size of C1/C2/C3 support networks.\n binary_hidden_sizes_val: &binary_hidden_sizes_val [500]\n\n # C4 classifier hidden sizes.\n c4_hidden_sizes_val: &c4_hidden_sizes_val [500, 500]\n\n # This comes from inputs fusion.\n activation_size_val: &activation_size_val 1134\n\n # Activation size + 3 * support output size\n activation_support_size_val: &activation_support_size_val 1434\n\n batch_size: &batch_size 512\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: all\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv\n # Use workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: all\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 6: masks for all 5 classifiers #################\n\n # Add global variables.\n c1234_global_publisher:\n type: GlobalVariablePublisher\n priority: 6.0\n # Add input_size to globals.\n keys: [category_c1_word_to_ix, category_c2_word_to_ix, category_c3_word_to_ix, category_c4_word_to_ix, category_binary_word_to_ix, c123_support_output_size]\n values: [{\"C1\": 0}, {\"C2\": 0}, {\"C3\": 0}, {\"C4\": 0}, {\"BINARY\": 0}, *c123_support_output_size_val]\n\n # Sample masking based on categories.\n c1_string_to_mask:\n priority: 6.1\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n # Input: predicted categories by the \"question categorizer\" pipeline.\n strings: predicted_category_names\n masks: c1_masks\n\n c2_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c2_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c2_masks\n\n c3_string_to_mask:\n priority: 6.3\n type: StringToMask\n globals:\n word_mappings: category_c3_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c3_masks\n\n c4_string_to_mask:\n priority: 6.4\n type: StringToMask\n globals:\n word_mappings: category_c4_word_to_ix\n streams:\n strings: predicted_category_names\n masks: c4_masks\n\n binary_string_to_mask:\n priority: 6.5\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: predicted_category_names\n masks: binary_masks\n\n ################# PIPE 7: answers/targets for all 5 classifiers #################\n\n c1_answer_indexer:\n type: LabelIndexer\n priority: 7.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c1_without_yn_answers_ids\n globals:\n vocabulary_size: c1_without_yn_vocabulary_size\n word_mappings: c1_without_yn_word_mappings\n\n c2_answer_indexer:\n type: LabelIndexer\n priority: 7.2\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c2.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c2_answers_ids\n globals:\n vocabulary_size: c2_vocabulary_size\n word_mappings: c2_word_mappings\n\n c3_answer_indexer:\n type: LabelIndexer\n priority: 7.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c3.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c3_answers_ids\n globals:\n vocabulary_size: c3_vocabulary_size\n word_mappings: c3_word_mappings\n\n c4_answer_indexer:\n type: LabelIndexer\n priority: 7.4\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c4_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: c4_without_yn_answers_ids\n globals:\n vocabulary_size: c4_without_yn_vocabulary_size\n word_mappings: c4_without_yn_word_mappings\n\n binary_answer_indexer:\n type: LabelIndexer\n priority: 7.6\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: binary_yn_answers_ids\n globals:\n vocabulary_size: binary_yn_vocabulary_size\n word_mappings: binary_yn_word_mappings\n\n # Answer encoding for all classes.\n c1234_answer_indexer:\n priority: 7.7\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.all.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: c1234_binary_yn_vocabulary_size\n word_mappings: c1234_binary_yn_word_mappings\n\n\n ################# PIPE 8: C1 questions #################\n\n c1_support_network:\n priority: 8.1\n type: FeedForwardNetwork\n hidden: *c123_support_hidden_sizes_val\n dropout_rate: 0.5\n # This is support network, so we do not want softmax... do we?\n use_logsoftmax: false\n streams:\n inputs: concatenated_activations\n predictions: c1_support\n globals:\n input_size: concatenated_activations_size\n prediction_size: c123_support_output_size\n\n c1_answer_classifier:\n priority: 8.2\n type: FeedForwardNetwork\n # No hidden layer, only projection to answer + softmax!\n dropout_rate: 0.5\n streams:\n inputs: c1_support\n predictions: c1_predictions\n globals:\n input_size: c123_support_output_size\n prediction_size: c1_without_yn_vocabulary_size\n\n c1_nllloss:\n priority: 8.3\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c1_masks\n predictions: c1_predictions\n targets: c1_without_yn_answers_ids\n loss: c1_loss\n\n c1_precision_recall:\n priority: 8.4\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c1_masks\n predictions: c1_predictions\n targets: c1_without_yn_answers_ids\n globals:\n word_mappings: c1_without_yn_word_mappings\n statistics:\n precision: c1_precision\n recall: c1_recall\n f1score: c1_f1score\n\n c1_prediction_decoder:\n priority: 8.5\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c1_predictions\n outputs: c1_without_yn_prediction_labels\n globals:\n word_mappings: c1_without_yn_word_mappings\n\n\n ################# PIPE 9: C2 questions #################\n\n c2_support_network:\n priority: 9.1\n type: FeedForwardNetwork\n hidden: *c123_support_hidden_sizes_val\n dropout_rate: 0.5\n # This is support network, so we do not want softmax... do we?\n use_logsoftmax: false\n streams:\n inputs: concatenated_activations\n predictions: c2_support\n globals:\n input_size: concatenated_activations_size\n prediction_size: c123_support_output_size\n\n c2_answer_classifier:\n priority: 9.2\n type: FeedForwardNetwork\n # No hidden layer, only projection to answer + softmax!\n dropout_rate: 0.5\n streams:\n inputs: c2_support\n predictions: c2_predictions\n globals:\n input_size: c123_support_output_size\n prediction_size: c2_vocabulary_size\n\n c2_nllloss:\n priority: 9.3\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c2_masks\n predictions: c2_predictions\n targets: c2_answers_ids\n loss: c2_loss\n\n c2_precision_recall:\n priority: 9.4\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c2_masks\n predictions: c2_predictions\n targets: c2_answers_ids\n globals:\n word_mappings: c2_word_mappings\n statistics:\n precision: c2_precision\n recall: c2_recall\n f1score: c2_f1score\n\n c2_prediction_decoder:\n priority: 9.5\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c2_predictions\n outputs: c2_prediction_labels\n globals:\n word_mappings: c2_word_mappings\n\n\n ################# PIPE 10: C3 questions #################\n\n c3_support_network:\n priority: 10.1\n type: FeedForwardNetwork\n hidden: *c123_support_hidden_sizes_val\n dropout_rate: 0.5\n # This is support network, so we do not want softmax... do we?\n use_logsoftmax: false\n streams:\n inputs: concatenated_activations\n predictions: c3_support\n globals:\n input_size: concatenated_activations_size\n prediction_size: c123_support_output_size\n\n c3_answer_classifier:\n priority: 10.2\n type: FeedForwardNetwork\n # No hidden layer, only projection to answer + softmax!\n dropout_rate: 0.5\n streams:\n inputs: c3_support\n predictions: c3_predictions\n globals:\n input_size: c123_support_output_size\n prediction_size: c3_vocabulary_size\n\n c3_nllloss:\n priority: 10.3\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c3_masks\n predictions: c3_predictions\n targets: c3_answers_ids\n loss: c3_loss\n\n c3_precision_recall:\n priority: 10.4\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c3_masks\n predictions: c3_predictions\n targets: c3_answers_ids\n globals:\n word_mappings: c3_word_mappings\n statistics:\n precision: c3_precision\n recall: c3_recall\n f1score: c3_f1score\n\n c3_prediction_decoder:\n priority: 10.5\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c3_predictions\n outputs: c3_prediction_labels\n globals:\n word_mappings: c3_word_mappings\n\n\n ################# PIPE 11: C4 questions #################\n\n # Concatenate input fused activatons with three supporting streams.\n activation_support_concat:\n priority: 11.1\n type: ConcatenateTensor\n input_streams: [concatenated_activations,c1_support,c2_support,c3_support]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,*activation_size_val],[-1,*c123_support_output_size_val],[-1,*c123_support_output_size_val],[-1,*c123_support_output_size_val]]\n output_dims: [-1,*activation_support_size_val]\n streams:\n outputs: concatenated_activations_supports\n globals:\n output_size: concatenated_activations_supports_size\n\n\n # Model: FFN for C4 answering\n c4_answer_classifier:\n priority: 11.2\n type: FeedForwardNetwork\n hidden: *c4_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations_supports\n predictions: c4_predictions\n globals:\n input_size: concatenated_activations_supports_size\n prediction_size: c4_without_yn_vocabulary_size\n\n c4_nllloss:\n priority: 11.3\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: c4_masks\n predictions: c4_predictions\n targets: c4_without_yn_answers_ids\n loss: c4_loss\n\n c4_precision_recall:\n priority: 11.4\n type: PrecisionRecallStatistics\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: c4_masks\n predictions: c4_predictions\n targets: c4_without_yn_answers_ids\n globals:\n word_mappings: c4_without_yn_word_mappings\n statistics:\n precision: c4_precision\n recall: c4_recall\n f1score: c4_f1score\n\n c4_prediction_decoder:\n priority: 11.5\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: c4_predictions\n outputs: c4_without_yn_prediction_labels\n globals:\n word_mappings: c4_without_yn_word_mappings\n\n\n ################# PIPE 12: BINARY Y/N questions #################\n\n # Model: FFN for BINARY Y/N answering\n binary_answer_classifier:\n priority: 12.2\n type: FeedForwardNetwork\n hidden: *binary_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations_supports\n predictions: binary_predictions\n globals:\n input_size: concatenated_activations_supports_size\n prediction_size: binary_yn_vocabulary_size\n\n binary_nllloss:\n priority: 12.3\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n masks: binary_masks\n predictions: binary_predictions\n targets: binary_yn_answers_ids\n loss: binary_loss\n\n binary_precision_recall:\n priority: 12.4\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n use_masking: True\n streams:\n masks: binary_masks\n predictions: binary_predictions\n targets: binary_yn_answers_ids\n globals:\n word_mappings: binary_yn_word_mappings\n statistics:\n precision: binary_precision\n recall: binary_recall\n f1score: binary_f1score\n\n binary_prediction_decoder:\n priority: 12.5\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: binary_predictions\n outputs: binary_prediction_labels\n globals:\n word_mappings: binary_yn_word_mappings\n\n ################# PIPE 13: MERGE ANSWERS #################\n\n # Merge predictions.\n pipe9_merged_predictions:\n type: JoinMaskedPredictions\n priority: 13.1\n # Names of used input streams.\n input_prediction_streams: [c1_predictions, c2_predictions, c3_predictions, c4_predictions, binary_predictions]\n input_mask_streams: [c1_masks, c2_masks, c3_masks, c4_masks, binary_masks]\n input_word_mappings: [c1_without_yn_word_mappings, c2_word_mappings, c3_word_mappings, c4_without_yn_word_mappings, binary_yn_word_mappings]\n globals: \n output_word_mappings: c1234_binary_yn_word_mappings\n streams:\n output_strings: predicted_answers\n output_indices: merged_prediction_indices\n\n # Statistics.\n pipe9_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 13.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n globals:\n word_mappings: c1234_binary_yn_word_mappings\n streams:\n targets: answers_ids\n predictions: merged_prediction_indices\n statistics:\n precision: merged_precision\n recall: merged_recall\n f1score: merged_f1score\n\n # Viewers.\n viewer:\n priority: 13.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n c1_masks, c2_masks, c3_masks, c4_masks, binary_masks,\n answers,\n c1_without_yn_prediction_labels,\n c2_prediction_labels,\n c3_prediction_labels,\n c4_without_yn_prediction_labels,\n binary_prediction_labels,\n predicted_answers\n\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/tom/glove_lstm_resnet152_att_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model resnet152\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n #image_encoder_output_size_val: &image_encoder_output_size_val 100\n # INFO: this variable is not important, as we are using features in this pipeline!!\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type QuestionDrivenAttention\n # Options: LowRankBilinearPooling | QuestionDrivenAttention\n #question_image_fusion_size_val: &question_image_fusion_size_val 1124\n # INFO: this variable is set by QuestionDrivenAttention component!\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 1134\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n batch_size: &batch_size 256\n preload_images: &preload_images True\n num_workers: &num_workers 1\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size] #, image_encoder_output_size] #, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val] #, *image_encoder_output_size_val] #, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Attention + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n # Attention params.\n latent_size: 100\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n #hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,1124],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/tom/glove_lstm_resnet152_mcb_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model resnet152\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n image_encoder_output_size_val: &image_encoder_output_size_val 100\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type CompactBilinearPooling\n # Options: LowRankBilinearPooling | CompactBilinearPooling | QuestionDrivenAttention\n question_image_fusion_size_val: &question_image_fusion_size_val 100\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 110\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [100]\n\n batch_size: &batch_size 150\n preload_images: &preload_images False\n num_workers: &num_workers 3\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val, *image_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/tom/glove_lstm_vgg16_att_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model vgg16\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n #image_encoder_output_size_val: &image_encoder_output_size_val 100\n # INFO: this variable is not important, as we are using features in this pipeline!!\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type QuestionDrivenAttention\n # Options: LowRankBilinearPooling | QuestionDrivenAttention\n #question_image_fusion_size_val: &question_image_fusion_size_val 1124\n # INFO: this variable is set by QuestionDrivenAttention component!\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 1134\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n batch_size: &batch_size 300\n preload_images: &preload_images False\n num_workers: &num_workers 4\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size] #, image_encoder_output_size] #, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val] #, *image_encoder_output_size_val] #, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n return_feature_maps: True\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Attention + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n # Attention params.\n latent_size: 100\n num_attention_heads: 2\n streams:\n image_encodings: feature_maps\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n #hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,1124],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/tom/glove_lstm_vgg16_ewm_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model vgg16\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n image_encoder_output_size_val: &image_encoder_output_size_val 100\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type LowRankBilinearPooling\n # Options: LowRankBilinearPooling | ? (component: question_image_fusion)\n question_image_fusion_size_val: &question_image_fusion_size_val 100\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 110\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [100]\n\n batch_size: &batch_size 100\n preload_images: &preload_images True\n num_workers: &num_workers 1\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val, *image_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/evaluation/tom/glove_lstm_vgg16_mcb_is_cat_ffn_c123_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/default_vqa_med_2019.yml #,vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\nhyperparameters:\n # In here I am putting some of the hyperparameters from spreadsheet.\n\n question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize\n # Accepted formats: a,b,c or [a,b,c]\n # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all\n\n image_preprocessing: &image_preprocessing normalize\n # Accepted formats: a,b,c or [a,b,c]\n # none | random_affine | random_horizontal_flip | normalize | all\n\n # Image encoder.\n image_encoder_model: &image_encoder_model vgg16\n # Options: vgg16 | densenet121 | resnet152 | resnet50\n image_encoder_output_size_val: &image_encoder_output_size_val 1000\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type CompactBilinearPooling\n # Options: LowRankBilinearPooling | CompactBilinearPooling | \n question_image_fusion_size_val: &question_image_fusion_size_val 200\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 210\n\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n batch_size: &batch_size 200\n preload_images: &preload_images True\n num_workers: &num_workers 1\n\n# Training parameters:\ntraining:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images.\n preload_images: *preload_images\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Use four workers for loading images.\n dataloader:\n num_workers: *num_workers\n\n # Optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.0001\n\n # Terminal conditions:\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n episode_limit: 10000\n epoch_limit: -1\n\n# Validation parameters:\nvalidation:\n task:\n batch_size: *batch_size\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: *question_preprocessing\n image_preprocessing: *image_preprocessing \n # Preload images: false, as we will need them only once, at the end.\n preload_images: false\n streams: \n questions: tokenized_questions\n dataloader:\n num_workers: 1\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, fused_activation_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val, *image_encoder_output_size_val, *question_image_fusion_size_val]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n # Answer encoding.\n pipe1_all_answer_indexer:\n priority: 0.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe1_embeddings_size \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [*question_image_fusion_size_val]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,*question_image_fusion_size_val],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: *answer_classifier_hidden_sizes_val\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n predictions: pipe6_c123_predictions\n targets: answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions,\n category_names, predicted_category_names,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/extend_answers.yml", "content": "# This config is not a standalone config!\n# It adds new sections (sets) without samplers and components for saving answers that we can use for getting final answers.\n\ntraining_answers:\n task:\n type: &p_type VQAMED2019\n data_folder: &data_folder ~/data/vqa-med\n split: training\n categories: all\n resize_image: &resize_image [224, 224]\n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\nvalidation_answers:\n task:\n type: *p_type\n data_folder: *data_folder\n split: validation\n categories: all\n resize_image: *resize_image \n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\n\n# Testing parameters:\ntest_answers:\n task:\n type: *p_type \n data_folder: *data_folder\n split: test_answers\n resize_image: *resize_image \n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\n# Add component for exporting answers to files.\npipeline:\n disable: viewer,question_tokenizer\n # Viewers.\n viewer_extended:\n priority: 100.4\n type: StreamViewer\n sample_number: 0\n input_streams: \n indices,image_ids,tokenized_questions,\n category_names,predicted_category_names,\n answers,predicted_answers\n\n answer_exporter:\n priority: 100.5\n type: StreamFileExporter\n separator: '|'\n filename: 'answers.csv'\n export_separator_line_to_csv: True\n export_header_to_csv: True\n input_streams: \n indices,image_ids,tokenized_questions,\n category_names,predicted_category_names,\n answers,predicted_answers\n\n submission_exporter:\n priority: 100.6\n type: StreamFileExporter\n separator: '|'\n filename: 'submission.txt'\n input_streams: \n image_ids,\n predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/extend_answers_c4.yml", "content": "# This config is not a standalone config!\n# It adds new sections (sets) without samplers and components for saving answers that we can use for getting final answers.\n\ntraining_answers:\n task:\n type: &p_type VQAMED2019\n data_folder: &data_folder ~/data/vqa-med\n split: training\n categories: C4\n resize_image: &resize_image [224, 224]\n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\nvalidation_answers:\n task:\n type: *p_type\n data_folder: *data_folder\n split: validation\n categories: C4\n resize_image: *resize_image \n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\n\n# Add component for exporting answers to files.\npipeline:\n disable: viewer,question_tokenizer\n # Viewers.\n viewer_extended:\n priority: 100.4\n type: StreamViewer\n sample_number: 0\n input_streams: \n indices,image_ids,tokenized_questions,\n category_names,predicted_categories,\n answers,tokenized_answers,predicted_answers\n\n answer_exporter:\n priority: 100.5\n type: StreamFileExporter\n separator: '|'\n filename: 'answers.csv'\n export_separator_line_to_csv: True\n input_streams: \n indices,image_ids,tokenized_questions,\n category_names,predicted_categories,\n answers,tokenized_answers,predicted_answers\n\n submission_exporter:\n priority: 100.6\n type: StreamFileExporter\n separator: '|'\n filename: 'submission.txt'\n input_streams: \n image_ids,\n predicted_answers\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml", "content": "# Part of pipeline containing components constituting the \"Inputs Fusion\" pipeline.\n\n# Inputs:\n# streams:\n# * tokenized_questions\n# * images\n# * image_sizes\n\n# Outputs:\n# streams:\n# * concatenated_activations\n# globals:\n# * concatenated_activations_size\n\n# Freeze the models in \"Inputs Fusion\".\nfreeze_models: &freeze_models True\n\n\ncheckpoint: &checkpoint ~/image-clef-2019/experiments/c4_encoders/20190504_202441/checkpoints/glove_lstm_vgg16_att_is_cat_ffn_c123_loss_best.pt\n# Loaded checkpoint: 20190504_202441\n# ~/image-clef-2019/experiments/c4_encoders/20190504_202441/checkpoints/glove_lstm_vgg16_att_is_cat_ffn_c123_loss_best.pt\n# + Model 'pipe1_question_embeddings' [SentenceEmbeddings] params saved \n# + Model 'pipe1_lstm' [RecurrentNeuralNetwork] params saved \n# + Model 'image_encoder' [GenericImageEncoder] params saved \n# + Model 'image_size_encoder' [FeedForwardNetwork] params saved \n# + Model 'question_image_fusion' [QuestionDrivenAttention] params saved \n# + Model 'question_image_ffn' [FeedForwardNetwork] params saved \n\n# This one will be skipped, as this is C123 classifier!\n# + Model 'pipe6_c123_answer_classifier' [FeedForwardNetwork] params saved \n\npipe_if0_hyperparameters:\n # WARNING: as we are loading the pretrained pipeline, all those values must stay!\n\n # Image encoder.\n image_encoder_model: &image_encoder_model vgg16\n \n # Question encoder.\n question_encoder_embeddings: &question_encoder_embeddings glove.6B.50d.txt\n # Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled\n question_encoder_embeddings_size_val: &question_encoder_embeddings_size_val 50\n question_encoder_lstm_size_val: &question_encoder_lstm_size_val 50\n question_encoder_output_size_val: &question_encoder_output_size_val 100\n \n # Fusion I: image + question\n question_image_fusion_type_val: &question_image_fusion_type QuestionDrivenAttention\n\n # Image size encoder.\n image_size_encoder_output_size_val: &image_size_encoder_output_size_val 10\n\n # Fusion II: (image + question) + image size (must be = question_image_fusion_size_val + image_size_encoder_output_size_val)\n question_image_size_fusion_size_val: &question_image_size_fusion_size_val 1134\n\n\npipeline:\n\n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n pipe_if0_global_publisher:\n priority: 0.11\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size]\n values: [*question_encoder_output_size_val, 2, *image_size_encoder_output_size_val]\n\n # Statistics.\n pipe_if0_batch_size:\n priority: 0.12\n type: BatchSizeStatistics\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe_if1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: *question_encoder_embeddings_size_val\n pretrained_embeddings_file: *question_encoder_embeddings\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: pipe1_question_embeddings\n freeze: *freeze_models\n ###################\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions\n globals:\n embeddings_size: pipe_if1_embeddings_size \n \n # Model 2: question RNN\n pipe_if1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n hidden_size: *question_encoder_lstm_size_val\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: pipe1_lstm\n freeze: *freeze_models\n ###################\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: pipe_if1_embeddings_size\n prediction_size: question_encoder_output_size\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n pipe_if2_image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: *image_encoder_model\n return_feature_maps: True\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: image_encoder\n freeze: *freeze_models\n ###################\n streams:\n inputs: images\n outputs: feature_maps\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n pipe_if3_image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_logsoftmax: False\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: image_size_encoder\n freeze: *freeze_models\n ###################\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Attention + FF.\n pipe_if4_question_image_fusion:\n priority: 4.1\n type: *question_image_fusion_type\n dropout_rate: 0.5\n # Attention params.\n latent_size: 100\n num_attention_heads: 2\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: question_image_fusion\n freeze: *freeze_models\n ###################\n streams:\n image_encodings: feature_maps\n question_encodings: question_activations\n outputs: fused_activations\n globals:\n question_encoding_size: question_encoder_output_size\n output_size: fused_activation_size\n\n\n pipe_if4_question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n dropout_rate: 0.5\n use_logsoftmax: False\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: question_image_ffn\n freeze: *freeze_models\n ###################\n streams:\n inputs: fused_activations\n predictions: question_image_activations\n globals:\n input_size: fused_activation_size\n prediction_size: fused_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n pipe_if5_concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,1124],[-1,*image_size_encoder_output_size_val]]\n output_dims: [-1,*question_image_size_fusion_size_val]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n" }, { "path": "configs/vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml", "content": "# Part of pipeline containing components constituting the \"Question Categorizer\" pipeline.\n\n# Inputs:\n# * tokenized_questions\n# * category_ids (targets)\n\n# Outputs:\n# * predicted_categories (predictions)\n# * predicted_category_names\n\n# \"Question Categorizer\"\n# 0.51: pipe_qc_global_publisher\n# 0.52: pipe_qc_question_embeddings\n# 0.53: pipe_qc_lstm\n# 0.54: pipe_qc_classifier\n\n# Plus additional components for displaying/statistics:\n# 0.55: pipe_qc_category_decoder\n# 0.56: pipe_qc_category_accuracy\n\n# Loaded checkpoint: 20190508_000705\ncheckpoint: &checkpoint ~/image-clef-2019/experiments/q_categorization/20190508_000705/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n\npipeline:\n ################# PIPE: QUESTION CATEGORIZATION #################\n\n # Add global variables - the ones related to only question categorization.\n pipe_qc_global_publisher:\n priority: 0.51\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [pipe_qc_question_encoder_output_size]\n values: [100]\n\n # Model 1: question embeddings\n pipe_qc_question_embeddings:\n priority: 0.52\n type: SentenceEmbeddings\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe_qc_embedded_questions\n globals:\n embeddings_size: pipe_qc_embeddings_size\n \n # Model 2: question RNN\n pipe_qc_lstm:\n priority: 0.53\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe_qc_embedded_questions\n predictions: pipe_qc_question_activations\n globals:\n input_size: pipe_qc_embeddings_size\n prediction_size: pipe_qc_question_encoder_output_size\n\n # Model 3: FFN question category\n pipe_qc_classifier:\n priority: 0.54\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: *checkpoint\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.7\n streams:\n inputs: pipe_qc_question_activations\n predictions: predicted_categories\n globals:\n input_size: pipe_qc_question_encoder_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe_qc_category_decoder:\n priority: 0.55\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predicted_categories\n outputs: predicted_category_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe_qc_category_accuracy:\n priority: 0.56\n type: AccuracyStatistics\n streams:\n targets: category_ids\n predictions: predicted_categories\n statistics:\n accuracy: categorization_accuracy\n " }, { "path": "configs/vqa_med_2019/frozen_pipelines/frozen_word_answer_glove_sum.yml", "content": "" }, { "path": "configs/vqa_med_2019/frozen_pipelines/input_fusion_processor_io.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs:\n vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml,\n vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml\n\n\ntraining_validation:\n task:\n type: &p_type VQAMED2019\n data_folder: &data_folder ~/data/vqa-med\n split: training_validation\n categories: all\n resize_image: &resize_image [224, 224]\n batch_size: 64\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n dataloader:\n # No sampler, process samples in the same order.\n shuffle: false\n # Use 1 worker, so batches will follow the samples order.\n num_workers: 1\n\n\nhyperparams:\n # Final classifier: FFN.\n answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]\n\n\n# Add component for exporting answers to files.\npipeline:\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Answer encoding.\n pipe6_c123_binary_yn_answer_indexer:\n priority: 6.2\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n\n # Viewers.\n viewer_extended:\n priority: 100.4\n type: StreamViewer\n sample_number: 0\n input_streams: \n indices,image_ids,tokenized_questions,\n concatenated_activations_size,\n category,names,\n answers\n\n fused_inputs_exporter:\n priority: 100.5\n type: StreamFileExporter\n separator: '|'\n filename: 'fused_inputs.csv'\n export_separator_line_to_csv: True\n input_streams: \n indices #, concatenated_activations\n\n" }, { "path": "configs/vqa_med_2019/question_categorization/default_question_categorization.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\ntraining:\n task:\n categories: all\n export_sample_weights: ~/data/vqa-med/answers.all.weights.csv\n # Do not load and stream images!\n stream_images: False\n batch_size: 256\n sampler:\n weights: ~/data/vqa-med/answers.all.weights.csv\n terminal_conditions:\n loss_stop_threshold: 1.0e-3\n\nvalidation:\n task:\n categories: all\n # Do not load and stream images!\n stream_images: False\n batch_size: 256\n\n\npipeline:\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 10.1\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_category_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 10.2\n targets_dim: 1\n streams:\n targets: category_ids\n loss: loss\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.1\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.2\n # streams:\n # targets: category_ids\n\n precision_recall:\n type: PrecisionRecallStatistics\n priority: 100.3\n use_word_mappings: True\n show_class_scores: True\n streams:\n targets: category_ids\n globals:\n num_classes: num_categories\n word_mappings: category_word_mappings\n \n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.4\n input_streams: questions,category_names,predicted_category_names\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/question_categorization/question_categorization_onehot_bow.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/question_categorization/default_question_categorization.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: question_vocabulary_size\n\n bow_encoder:\n type: BOWEncoder\n priority: 1.3\n streams:\n inputs: encoded_questions\n outputs: bow_questions\n globals:\n bow_size: question_vocabulary_size # Set by question_encoder.\n\n # Model\n classifier:\n type: FeedForwardNetwork\n #freeze: True\n priority: 3\n streams:\n inputs: bow_questions\n globals:\n input_size: question_vocabulary_size # Set by question_encoder.\n prediction_size: num_categories # C1,C2,C3,C4\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 4\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_categories\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n targets_dim: 1\n streams:\n targets: category_ids\n loss: loss\n\n # Statistics.\n accuracy:\n type: AccuracyStatistics\n priority: 10\n streams:\n targets: category_ids\n batch_size:\n type: BatchSizeStatistics\n priority: 11\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 12\n stream_names: questions,category_names,predicted_categories\n\n #: pipeline\n" }, { "path": "configs/vqa_med_2019/question_categorization/question_categorization_onehot_rnn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/question_categorization/default_question_categorization.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n question_encoder:\n type: SentenceOneHotEncoder\n priority: 1.2\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_questions\n outputs: encoded_questions\n globals:\n vocabulary_size: question_vocabulary_size\n\n question_to_tensor:\n type: ListToTensor\n priority: 1.3\n num_inputs_dims: 3\n streams:\n inputs: encoded_questions\n outputs: tensor_questions\n globals:\n input_size: question_vocabulary_size\n\n # Model 1: RNN\n lstm:\n type: RecurrentNeuralNetwork\n prediction_mode: Last\n priority: 3\n initial_state: Zero\n hidden_size: 50\n streams:\n inputs: tensor_questions\n globals:\n input_size: question_vocabulary_size\n prediction_size: num_categories # C1,C2,C3,C4\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/question_categorization/question_categorization_rnn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/question_categorization/default_question_categorization.yml\n\npipeline:\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 2: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 3\n initial_state: Zero\n streams:\n inputs: embedded_questions\n globals:\n input_size: embeddings_size\n prediction_size: num_categories # C1,C2,C3,C4,BIN,UNK\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/question_categorization/question_categorization_rnn_ffn.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/question_categorization/default_question_categorization.yml\n\npipeline:\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]\n values: [100, 2, 10, 100]\n\n\n # Questions encoding.\n question_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 2: Embeddings\n question_embeddings:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n priority: 1.3\n initial_state: Zero\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: lstm_activations_questions\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model\n classifier:\n type: FeedForwardNetwork\n hidden: [50]\n priority: 1.4\n dropout_rate: 0.5\n streams:\n inputs: lstm_activations_questions\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/c1_binary_vf_cat_hard_shared_question_rnn_two_ffns_losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1\n export_sample_weights: ~/data/vqa-med/answers.c1.weights.csv\n sampler:\n type: kFoldWeightedRandomSampler\n weights: ~/data/vqa-med/answers.c1.weights.csv\n folds: 5\n dataloader:\n num_workers: 4\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1\n sampler:\n type: kFoldRandomSampler\n folds: 5\n dataloader:\n num_workers: 4\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_word_to_ix, category_binary_word_to_ix]\n values: [100, 2, 10, 100,{\"C1\": 0}, {\"BINARY\": 0}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: SHARED QUESTION #################\n\n # Questions encoding.\n pipe0_question_tokenizer:\n priority: 0.2\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n #load: \n # file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n # model: question_embeddings\n #freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n #load: \n # file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n # model: lstm\n #freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: lstm_activations_questions\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n ################# PIPE 1: CATEGORY #################\n\n ################# PIPE 2: C1 question #################\n\n # Answer encoding for pipe 2.\n pipe2_c1_answer_indexer:\n type: LabelIndexer\n priority: 2.0\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe2_c1_answers_without_yn_ids\n globals:\n vocabulary_size: vocabulary_size_c1_without_yn\n word_mappings: word_mappings_c1_without_yn\n\n # Sample masking based on categories.\n pipe2_c1_string_to_mask:\n priority: 2.1\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n strings: category_names # predicted_question_categories_names ## FOR NOW!\n string_indices: predicted_c1_question_categories_indices # NOT USED\n masks: pipe2_c1_masks\n\n # Model 4: FFN C1 answering\n pipe2_c1_lstm:\n priority: 2.2\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: lstm_activations_questions\n predictions: pipe2_c1_predictions\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: vocabulary_size_c1_without_yn\n\n pipe2_c1_nllloss:\n type: NLLLoss\n priority: 2.3\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe2_c1_predictions\n masks: pipe2_c1_masks\n targets: pipe2_c1_answers_without_yn_ids\n loss: pipe2_c1_loss\n\n pipe2_c1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 2.4\n use_masking: True\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe2_c1_masks\n predictions: pipe2_c1_predictions\n targets: pipe2_c1_answers_without_yn_ids\n globals:\n word_mappings: word_mappings_c1_without_yn\n num_classes: vocabulary_size_c1_without_yn\n statistics:\n precision: pipe2_c1_precision\n recall: pipe2_c1_recall\n f1score: pipe2_c1_f1score\n\n ################# PIPE 3: BINARY question #################\n\n # Answer encoding for pipe 3.\n pipe3_binary_answer_indexer:\n type: LabelIndexer\n priority: 3.0\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe3_binary_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n pipe3_binary_string_to_mask:\n priority: 3.1\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: category_names # predicted_question_categories_names ## FOR NOW!\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe3_binary_masks\n\n # Model 4: FFN C1 answering\n pipe3_binary_lstm:\n priority: 3.2\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: lstm_activations_questions\n predictions: pipe3_binary_predictions\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: vocabulary_size_binary_yn\n\n pipe3_binary_nllloss:\n type: NLLLoss\n priority: 3.3\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe3_binary_predictions\n masks: pipe3_binary_masks\n targets: pipe3_binary_answers_ids\n loss: pipe3_binary_loss\n\n pipe3_binary_precision_recall:\n type: PrecisionRecallStatistics\n priority: 3.4\n use_masking: True\n use_word_mappings: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe3_binary_masks\n predictions: pipe3_binary_predictions\n targets: pipe3_binary_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n num_classes: vocabulary_size_binary_yn\n statistics:\n precision: pipe3_binary_precision\n recall: pipe3_binary_recall\n f1score: pipe3_binary_f1score\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.1\n input_streams: questions,answers, category_names,predicted_question_categories_names, pipe2_c1_masks,pipe2_c1_answers_without_yn_ids,pipe2_c1_predictions, pipe3_binary_masks,pipe3_binary_answers_ids,pipe3_binary_predictions\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/c1_binary_vf_cat_rnn_shared_all_encoders_two_ffns_losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1\n export_sample_weights: ~/data/vqa-med/answers.c1.weights.csv\n sampler:\n type: kFoldWeightedRandomSampler\n weights: ~/data/vqa-med/answers.c1.weights.csv\n folds: 5\n dataloader:\n num_workers: 4\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1\n sampler:\n type: kFoldRandomSampler\n folds: 5\n dataloader:\n num_workers: 4\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_word_to_ix, category_binary_word_to_ix]\n values: [100, 2, 10, 100,{\"C1\": 0}, {\"BINARY\": 0}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n # Questions encoding.\n pipe1_question_tokenizer:\n priority: 0.2\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_all_c1_binasry\n word_mappings: word_mappings_all_c1_binary\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 question #################\n\n # Answer encoding for PIPE 5.\n pipe5_c1_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c1_answers_without_yn_ids\n globals:\n vocabulary_size: vocabulary_size_c1_without_yn\n word_mappings: word_mappings_c1_without_yn\n\n # Sample masking based on categories.\n pipe5_c1_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c1_question_categories_indices # NOT USED\n masks: pipe5_c1_masks\n\n # Model 4: FFN C1 answering\n pipe5_c1_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c1_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c1_without_yn\n\n pipe5_c1_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c1_predictions\n masks: pipe5_c1_masks\n targets: pipe5_c1_answers_without_yn_ids\n loss: pipe5_c1_loss\n\n pipe5_c1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c1_masks\n predictions: pipe5_c1_predictions\n targets: pipe5_c1_answers_without_yn_ids\n globals:\n word_mappings: word_mappings_c1_without_yn\n num_classes: vocabulary_size_c1_without_yn\n statistics:\n precision: pipe5_c1_precision\n recall: pipe5_c1_recall\n f1score: pipe5_c1_f1score\n\n ################# PIPE 6: BINARY question #################\n\n # Answer encoding for pipe 6.\n pipe6_binary_answer_indexer:\n type: LabelIndexer\n priority: 6.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe6_binary_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n pipe6_binary_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe6_binary_masks\n\n # Model 4: FFN C1 answering\n pipe6_binary_ffn:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_binary_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_binary_yn\n\n pipe6_binary_nllloss:\n type: NLLLoss\n priority: 6.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe6_binary_predictions\n masks: pipe6_binary_masks\n targets: pipe6_binary_answers_ids\n loss: pipe6_binary_loss\n\n pipe6_binary_precision_recall:\n type: PrecisionRecallStatistics\n priority: 6.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe6_binary_masks\n predictions: pipe6_binary_predictions\n targets: pipe6_binary_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n num_classes: vocabulary_size_binary_yn\n statistics:\n precision: pipe6_binary_precision\n recall: pipe6_binary_recall\n f1score: pipe6_binary_f1score\n\n ################# PIPE 7: MERGE ANSWERS #################\n\n # Merge predictions\n pipe7_merged_predictions:\n type: JoinMaskedPredictions\n priority: 7.1\n # Names of used input streams.\n input_prediction_streams: [pipe5_c1_predictions, pipe6_binary_predictions]\n input_mask_streams: [pipe5_c1_masks, pipe6_binary_masks]\n input_word_mappings: [word_mappings_c1_without_yn, word_mappings_binary_yn]\n globals:\n output_word_mappings: word_mappings_all_c1_binary\n streams:\n output_strings: pipe7_merged_predictions\n output_indices: pipe7_merged_pred_indices\n\n # Statistics.\n pipe7_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 7.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: word_mappings_all_c1_binary\n streams:\n targets: all_answers_ids\n predictions: pipe7_merged_pred_indices\n statistics:\n precision: pipe7_merged_precision\n recall: pipe7_merged_recall\n f1score: pipe7_merged_f1score\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 7.3\n input_streams: questions,answers, category_names,predicted_question_categories_names, pipe5_c1_masks,pipe5_c1_answers_without_yn_ids,pipe5_c1_predictions, pipe6_binary_masks,pipe6_binary_answers_ids,pipe6_binary_predictions, pipe7_merged_predictions\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/c1_binary_vf_cat_rnn_shared_question_rnn_two_ffns_losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1\n export_sample_weights: ~/data/vqa-med/answers.c1.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c1.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_word_to_ix, category_binary_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0}, {\"BINARY\": 0}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: SHARED QUESTION #################\n\n # Questions encoding.\n pipe0_question_tokenizer:\n priority: 0.2\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n #load: \n # file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n # model: question_embeddings\n #freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n #load: \n # file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n # model: lstm\n #freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: lstm_activations_questions\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe0_all_answer_indexer:\n type: LabelIndexer\n priority: 0.6\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_all_c1_binasry\n word_mappings: word_mappings_all_c1_binary\n\n\n ################# PIPE 1: CATEGORY #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe1_embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: pipe1_embedded_questions\n predictions: pipe1_lstm_activations_questions\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe1_classifier:\n priority: 1.3\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe1_lstm_activations_questions\n predictions: pipe1_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe1_category_decoder:\n priority: 1.4\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe1_predicted_question_categories_preds\n outputs: pipe1_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe1_category_accuracy:\n type: AccuracyStatistics\n priority: 1.5\n streams:\n targets: category_ids\n predictions: pipe1_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n\n ################# PIPE 2: C1 question #################\n\n # Answer encoding for PIPE 2.\n pipe2_c1_answer_indexer:\n type: LabelIndexer\n priority: 2.0\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe2_c1_answers_without_yn_ids\n globals:\n vocabulary_size: vocabulary_size_c1_without_yn\n word_mappings: word_mappings_c1_without_yn\n\n # Sample masking based on categories.\n pipe2_c1_string_to_mask:\n priority: 2.1\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n strings: pipe1_predicted_question_categories_names\n string_indices: predicted_c1_question_categories_indices # NOT USED\n masks: pipe2_c1_masks\n\n # Model 4: FFN C1 answering\n pipe2_c1_lstm:\n priority: 2.2\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: lstm_activations_questions\n predictions: pipe2_c1_predictions\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: vocabulary_size_c1_without_yn\n\n pipe2_c1_nllloss:\n type: NLLLoss\n priority: 2.3\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe2_c1_predictions\n masks: pipe2_c1_masks\n targets: pipe2_c1_answers_without_yn_ids\n loss: pipe2_c1_loss\n\n pipe2_c1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 2.4\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe2_c1_masks\n predictions: pipe2_c1_predictions\n targets: pipe2_c1_answers_without_yn_ids\n globals:\n word_mappings: word_mappings_c1_without_yn\n num_classes: vocabulary_size_c1_without_yn\n statistics:\n precision: pipe2_c1_precision\n recall: pipe2_c1_recall\n f1score: pipe2_c1_f1score\n\n ################# PIPE 3: BINARY question #################\n\n # Answer encoding for PIPE 3.\n pipe3_binary_answer_indexer:\n type: LabelIndexer\n priority: 3.0\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe3_binary_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n pipe3_binary_string_to_mask:\n priority: 3.1\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: pipe1_predicted_question_categories_names\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe3_binary_masks\n\n # Model 4: FFN C1 answering\n pipe3_binary_lstm:\n priority: 3.2\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: lstm_activations_questions\n predictions: pipe3_binary_predictions\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: vocabulary_size_binary_yn\n\n pipe3_binary_nllloss:\n type: NLLLoss\n priority: 3.3\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe3_binary_predictions\n masks: pipe3_binary_masks\n targets: pipe3_binary_answers_ids\n loss: pipe3_binary_loss\n\n pipe3_binary_precision_recall:\n type: PrecisionRecallStatistics\n priority: 3.4\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe3_binary_masks\n predictions: pipe3_binary_predictions\n targets: pipe3_binary_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n num_classes: vocabulary_size_binary_yn\n statistics:\n precision: pipe3_binary_precision\n recall: pipe3_binary_recall\n f1score: pipe3_binary_f1score\n\n ################# PIPE 4: MERGE ANSWERS #################\n\n # Merge predictions\n merged_predictions:\n type: JoinMaskedPredictions\n priority: 4.1\n # Names of used input streams.\n input_prediction_streams: [pipe2_c1_predictions, pipe3_binary_predictions]\n input_mask_streams: [pipe2_c1_masks, pipe3_binary_masks]\n input_word_mappings: [word_mappings_c1_without_yn, word_mappings_binary_yn]\n globals:\n output_word_mappings: word_mappings_all_c1_binary\n streams:\n output_strings: merged_predictions\n output_indices: merged_pred_indices\n\n # Statistics.\n merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 4.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: word_mappings_all_c1_binary\n streams:\n targets: all_answers_ids\n predictions: merged_pred_indices\n statistics:\n precision: merged_precision\n recall: merged_recall\n f1score: merged_f1score\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 4.3\n input_streams: questions,answers, category_names,predicted_question_categories_names, pipe2_c1_masks,pipe2_c1_answers_without_yn_ids,pipe2_c1_predictions, pipe3_binary_masks,pipe3_binary_answers_ids,pipe3_binary_predictions, merged_predictions\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/c1_c2_c3_binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_word_to_ix, category_c2_word_to_ix, category_c3_word_to_ix, category_binary_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0}, {\"C2\": 0}, {\"C3\": 0}, {\"BINARY\": 0}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n # Questions encoding.\n pipe1_question_tokenizer:\n priority: 0.2\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n #vocabulary_size: vocabulary_size_all_c1_c2_c3_binary\n word_mappings: word_mappings_all_c1_c2_c3_binary\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 question #################\n\n # Answer encoding for PIPE 5.\n pipe5_c1_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c1_answers_without_yn_ids\n globals:\n vocabulary_size: vocabulary_size_c1_without_yn\n word_mappings: word_mappings_c1_without_yn\n\n # Sample masking based on categories.\n pipe5_c1_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c1_question_categories_indices # NOT USED\n masks: pipe5_c1_masks\n\n # Model 4: FFN C1 answering\n pipe5_c1_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c1_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c1_without_yn\n\n pipe5_c1_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c1_predictions\n masks: pipe5_c1_masks\n targets: pipe5_c1_answers_without_yn_ids\n loss: pipe5_c1_loss\n\n pipe5_c1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c1_masks\n predictions: pipe5_c1_predictions\n targets: pipe5_c1_answers_without_yn_ids\n globals:\n word_mappings: word_mappings_c1_without_yn\n #num_classes: vocabulary_size_c1_without_yn\n statistics:\n precision: pipe5_c1_precision\n recall: pipe5_c1_recall\n f1score: pipe5_c1_f1score\n\n ################# PIPE 6: C2 question #################\n\n pipe6_c2_answer_indexer:\n type: LabelIndexer\n priority: 6.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c2.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe6_c2_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c2\n word_mappings: word_mappings_c2\n\n # Sample masking based on categories.\n pipe6_c2_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c2_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c2_question_categories_indices # NOT USED\n masks: pipe6_c2_masks\n\n pipe6_c2_ffn:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c2_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c2\n\n pipe6_c2_nllloss:\n type: NLLLoss\n priority: 6.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe6_c2_predictions\n masks: pipe6_c2_masks\n targets: pipe6_c2_answers_ids\n loss: pipe6_c2_loss\n\n pipe6_c2_precision_recall:\n type: PrecisionRecallStatistics\n priority: 6.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe6_c2_masks\n predictions: pipe6_c2_predictions\n targets: pipe6_c2_answers_ids\n globals:\n word_mappings: word_mappings_c2\n statistics:\n precision: pipe6_c2_precision\n recall: pipe6_c2_recall\n f1score: pipe6_c2_f1score\n\n ################# PIPE 7: C3 question #################\n\n # Answer encoding for PIPE 7.\n pipe7_c3_answer_indexer:\n type: LabelIndexer\n priority: 7.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c3.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe7_c3_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c3\n word_mappings: word_mappings_c3\n\n # Sample masking based on categories.\n pipe7_c3_string_to_mask:\n priority: 7.2\n type: StringToMask\n globals:\n word_mappings: category_c3_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c3_question_categories_indices # NOT USED\n masks: pipe7_c3_masks\n\n # Model 4: FFN C1 answering\n pipe7_c3_ffn:\n priority: 7.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe7_c3_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c3\n\n pipe7_c3_nllloss:\n type: NLLLoss\n priority: 7.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe7_c3_predictions\n masks: pipe7_c3_masks\n targets: pipe7_c3_answers_ids\n loss: pipe7_c3_loss\n\n pipe7_c3_precision_recall:\n type: PrecisionRecallStatistics\n priority: 7.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe7_c3_masks\n predictions: pipe7_c3_predictions\n targets: pipe7_c3_answers_ids\n globals:\n word_mappings: word_mappings_c3\n #num_classes: vocabulary_size_c3\n statistics:\n precision: pipe7_c3_precision\n recall: pipe7_c3_recall\n f1score: pipe7_c3_f1score\n\n ################# PIPE 8: BINARY question #################\n\n # Answer encoding for pipe 8.\n pipe8_binary_answer_indexer:\n type: LabelIndexer\n priority: 8.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe8_binary_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n pipe8_binary_string_to_mask:\n priority: 8.2\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe8_binary_masks\n\n # Model 4: FFN C1 answering\n pipe8_binary_ffn:\n priority: 8.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe8_binary_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_binary_yn\n\n pipe8_binary_nllloss:\n type: NLLLoss\n priority: 8.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe8_binary_predictions\n masks: pipe8_binary_masks\n targets: pipe8_binary_answers_ids\n loss: pipe8_binary_loss\n\n pipe8_binary_precision_recall:\n type: PrecisionRecallStatistics\n priority: 8.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe8_binary_masks\n predictions: pipe8_binary_predictions\n targets: pipe8_binary_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n #num_classes: vocabulary_size_binary_yn\n statistics:\n precision: pipe8_binary_precision\n recall: pipe8_binary_recall\n f1score: pipe8_binary_f1score\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Merge predictions.\n pipe9_merged_predictions:\n type: JoinMaskedPredictions\n priority: 9.1\n # Names of used input streams.\n input_prediction_streams: [pipe5_c1_predictions, pipe6_c2_predictions, pipe7_c3_predictions, pipe8_binary_predictions]\n input_mask_streams: [pipe5_c1_masks, pipe6_c2_masks, pipe7_c3_masks, pipe8_binary_masks]\n input_word_mappings: [word_mappings_c1_without_yn, word_mappings_c2, word_mappings_c3, word_mappings_binary_yn]\n globals: \n output_word_mappings: word_mappings_all_c1_c2_c3_binary\n streams:\n output_strings: pipe9_merged_predictions\n output_indices: pipe9_merged_pred_indices\n\n # Statistics.\n pipe9_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 9.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: word_mappings_all_c1_c2_c3_binary\n streams:\n targets: all_answers_ids\n predictions: pipe9_merged_pred_indices\n statistics:\n precision: pipe9_merged_precision\n recall: pipe9_merged_recall\n f1score: pipe9_merged_f1score\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams: questions,answers, category_names,predicted_question_categories_names, pipe5_c1_masks,pipe5_c1_answers_without_yn_ids,pipe5_c1_predictions, pipe6_c2_masks,pipe6_c2_answers_ids,pipe6_c2_predictions, pipe7_c3_masks,pipe7_c3_answers_ids,pipe7_c3_predictions, pipe8_binary_masks,pipe8_binary_answers_ids,pipe8_binary_predictions, pipe9_merged_predictions\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/c1_c3_binary_vf_cat_rnn_shared_all_encoders_three_ffns_losses.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c3_binary_yn.weights.csv\n sampler:\n weights: ~/data/vqa-med/answers.c1_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C3\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_word_to_ix, category_c3_word_to_ix, category_binary_word_to_ix]\n values: [100, 2, 10, 100,{\"C1\": 0}, {\"C3\": 0}, {\"BINARY\": 0}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n # Questions encoding.\n pipe1_question_tokenizer:\n priority: 0.2\n type: SentenceTokenizer\n streams: \n inputs: questions\n outputs: tokenized_questions\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n\n\n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n dropout_rate: 0.5\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n #vocabulary_size: vocabulary_size_all_c1_c3_binary\n word_mappings: word_mappings_all_c1_c3_binary\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 question #################\n\n # Answer encoding for PIPE 5.\n pipe5_c1_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c1_answers_without_yn_ids\n globals:\n vocabulary_size: vocabulary_size_c1_without_yn\n word_mappings: word_mappings_c1_without_yn\n\n # Sample masking based on categories.\n pipe5_c1_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c1_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c1_question_categories_indices # NOT USED\n masks: pipe5_c1_masks\n\n # Model 4: FFN C1 answering\n pipe5_c1_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c1_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c1_without_yn\n\n pipe5_c1_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c1_predictions\n masks: pipe5_c1_masks\n targets: pipe5_c1_answers_without_yn_ids\n loss: pipe5_c1_loss\n\n pipe5_c1_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c1_masks\n predictions: pipe5_c1_predictions\n targets: pipe5_c1_answers_without_yn_ids\n globals:\n word_mappings: word_mappings_c1_without_yn\n #num_classes: vocabulary_size_c1_without_yn\n statistics:\n precision: pipe5_c1_precision\n recall: pipe5_c1_recall\n f1score: pipe5_c1_f1score\n\n ################# PIPE 6: C3 question #################\n\n # Answer encoding for PIPE 6.\n pipe6_c3_answer_indexer:\n type: LabelIndexer\n priority: 6.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c3.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe6_c3_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c3\n word_mappings: word_mappings_c3\n\n # Sample masking based on categories.\n pipe6_c3_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c3_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c3_question_categories_indices # NOT USED\n masks: pipe6_c3_masks\n\n # Model 4: FFN C1 answering\n pipe6_c3_ffn:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c3_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c3\n\n pipe6_c3_nllloss:\n type: NLLLoss\n priority: 6.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe6_c3_predictions\n masks: pipe6_c3_masks\n targets: pipe6_c3_answers_ids\n loss: pipe6_c3_loss\n\n pipe6_c3_precision_recall:\n type: PrecisionRecallStatistics\n priority: 6.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe6_c3_masks\n predictions: pipe6_c3_predictions\n targets: pipe6_c3_answers_ids\n globals:\n word_mappings: word_mappings_c3\n #num_classes: vocabulary_size_c3\n statistics:\n precision: pipe6_c3_precision\n recall: pipe6_c3_recall\n f1score: pipe6_c3_f1score\n\n ################# PIPE 7: BINARY question #################\n\n # Answer encoding for pipe 7.\n pipe7_binary_answer_indexer:\n type: LabelIndexer\n priority: 7.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe7_binary_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n pipe7_binary_string_to_mask:\n priority: 7.2\n type: StringToMask\n globals:\n word_mappings: category_binary_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe7_binary_masks\n\n # Model 4: FFN C1 answering\n pipe7_binary_ffn:\n priority: 7.3\n type: FeedForwardNetwork\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe7_binary_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_binary_yn\n\n pipe7_binary_nllloss:\n type: NLLLoss\n priority: 7.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe7_binary_predictions\n masks: pipe7_binary_masks\n targets: pipe7_binary_answers_ids\n loss: pipe7_binary_loss\n\n pipe7_binary_precision_recall:\n type: PrecisionRecallStatistics\n priority: 7.5\n use_word_mappings: True\n use_masking: True\n #show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe7_binary_masks\n predictions: pipe7_binary_predictions\n targets: pipe7_binary_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n #num_classes: vocabulary_size_binary_yn\n statistics:\n precision: pipe7_binary_precision\n recall: pipe7_binary_recall\n f1score: pipe7_binary_f1score\n\n ################# PIPE 8: MERGE ANSWERS #################\n\n # Merge predictions\n pipe8_merged_predictions:\n type: JoinMaskedPredictions\n priority: 8.1\n # Names of used input streams.\n input_prediction_streams: [pipe5_c1_predictions, pipe6_c3_predictions, pipe7_binary_predictions]\n input_mask_streams: [pipe5_c1_masks, pipe6_c3_masks, pipe7_binary_masks]\n input_word_mappings: [word_mappings_c1_without_yn, word_mappings_c3, word_mappings_binary_yn]\n globals:\n output_word_mappings: word_mappings_all_c1_c3_binary\n streams:\n output_strings: pipe8_merged_predictions\n output_indices: pipe8_merged_pred_indices\n\n # Statistics.\n pipe8_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 8.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: word_mappings_all_c1_c3_binary\n streams:\n targets: all_answers_ids\n predictions: pipe8_merged_pred_indices\n statistics:\n precision: pipe8_merged_precision\n recall: pipe8_merged_recall\n f1score: pipe8_merged_f1score\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 8.3\n input_streams: questions,answers, category_names,predicted_question_categories_names, pipe5_c1_masks,pipe5_c1_answers_without_yn_ids,pipe5_c1_predictions, pipe6_c3_masks,pipe6_c3_answers_ids,pipe6_c3_predictions, pipe7_binary_masks,pipe7_binary_answers_ids,pipe7_binary_predictions, pipe8_merged_predictions\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_resnet152_is_cat_ffn_c123_no_binary_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c123_without_yn_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n model: resnet152\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 + C2 + C3 questions #################\n\n # Answer encoding for PIPE 5.\n pipe5_c123_without_yn_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c123_without_yn_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n # Sample masking based on categories.\n pipe5_c123_without_yn_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c123_without_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe5_c123_without_yn_masks\n\n # Model 4: FFN C1 answering\n pipe5_c123_without_yn_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c123_without_yn_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_without_yn\n\n pipe5_c123_without_yn_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c123_without_yn_predictions\n masks: pipe5_c123_without_yn_masks\n targets: pipe5_c123_without_yn_answers_ids\n loss: pipe5_c123_without_yn_loss\n\n pipe5_c123_without_yn_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c123_without_yn_masks\n predictions: pipe5_c123_without_yn_predictions\n targets: pipe5_c123_without_yn_answers_ids\n globals:\n word_mappings: word_mappings_c123_without_yn\n statistics:\n precision: pipe5_c123_without_yn_precision\n recall: pipe5_c123_without_yn_recall\n f1score: pipe5_c123_without_yn_f1score\n\n # C123 Predictions decoder.\n pipe5_prediction_decoder:\n type: WordDecoder\n priority: 5.6\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe5_c123_without_yn_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe5_c123_without_yn_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_resnet50_ewm_is_cat_ffn_c123_loss_ffn_yn_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, element_wise_activation_size, category_c123_without_yn_word_to_ix,category_binary_yn_word_to_ix]\n values: [100, 2, 10, 100, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2}, {\"BINARY\": 3}]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n priority: 0.3\n type: SentenceEmbeddings\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_question_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_encoder_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n priority: 0.7\n type: AccuracyStatistics\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n priority: 1.3\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: resnet50\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Answer encoding for PIPE 6.\n pipe6_c123_answer_indexer:\n priority: 6.1\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe6_c123_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n # Sample masking based on categories.\n pipe6_c123_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c123_without_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe6_c123_masks\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_without_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe6_c123_predictions\n masks: pipe6_c123_masks\n targets: pipe6_c123_answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe6_c123_masks\n predictions: pipe6_c123_predictions\n targets: pipe6_c123_answers_ids\n globals:\n word_mappings: word_mappings_c123_without_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n #pipe5_c123_prediction_decoder:\n # priority: 6.6\n # type: WordDecoder\n # # Use the same word mappings as label indexer.\n # import_word_mappings_from_globals: True\n # streams:\n # inputs: pipe6_c123_predictions\n # outputs: pipe6_c123_predicted_answers\n # globals:\n # word_mappings: word_mappings_c123_without_yn\n\n\n ################# PIPE 7: Y/N questions #################\n\n # Answer encoding for PIPE 5.\n pipe7_binary_yn_answer_indexer:\n type: LabelIndexer\n priority: 7.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe7_binary_yn_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n # Sample masking based on categories.\n pipe7_binary_yn_string_to_mask:\n priority: 7.2\n type: StringToMask\n globals:\n word_mappings: category_binary_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_binary_question_categories_indices # NOT USED\n masks: pipe7_binary_yn_masks\n\n # Model 4: FFN C1 answering\n pipe7_binary_yn_classifier:\n priority: 7.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe7_binary_yn_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_binary_yn\n\n pipe7_binary_yn_nllloss:\n type: NLLLoss\n priority: 7.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe7_binary_yn_predictions\n masks: pipe7_binary_yn_masks\n targets: pipe7_binary_yn_answers_ids\n loss: pipe7_binary_yn_loss\n\n pipe7_binary_yn_precision_recall:\n type: PrecisionRecallStatistics\n priority: 7.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe7_binary_yn_masks\n predictions: pipe7_binary_yn_predictions\n targets: pipe7_binary_yn_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n statistics:\n precision: pipe7_binary_yn_precision\n recall: pipe7_binary_yn_recall\n f1score: pipe7_binary_yn_f1score\n\n # Y/N Predictions decoder.\n #pipe7_binary_yn_prediction_decoder:\n # type: WordDecoder\n # priority: 7.6\n # # Use the same word mappings as label indexer.\n # import_word_mappings_from_globals: True\n # streams:\n # inputs: pipe7_binary_yn_predictions\n # outputs: pipe7_binary_yn_predicted_answers\n # globals:\n # word_mappings: word_mappings_binary_yn\n\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Merge predictions\n pipe8_merged_predictions:\n type: JoinMaskedPredictions\n priority: 8.1\n # Names of used input streams.\n input_prediction_streams: [pipe6_c123_predictions, pipe7_binary_yn_predictions]\n input_mask_streams: [pipe6_c123_masks, pipe7_binary_yn_masks]\n input_word_mappings: [word_mappings_c123_without_yn, word_mappings_binary_yn]\n globals:\n output_word_mappings: word_mappings_c123_binary_yn\n streams:\n output_strings: predicted_answers\n output_indices: pipe8_merged_pred_indices\n\n # Statistics.\n pipe8_merged_precision_recall:\n type: PrecisionRecallStatistics\n priority: 8.2\n # Use prediction indices instead of distributions.\n use_prediction_distributions: False\n use_word_mappings: True\n show_class_scores: True\n show_confusion_matrix: True\n globals:\n word_mappings: word_mappings_c123_binary_yn\n streams:\n targets: all_answers_ids\n predictions: pipe8_merged_pred_indices\n statistics:\n precision: pipe8_merged_precision\n recall: pipe8_merged_recall\n f1score: pipe8_merged_f1score\n\n\n\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe6_c123_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_resnet50_ewm_is_cat_ffn_c123_no_binary_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n priority: 0\n type: GlobalVariablePublisher\n # Add input_size to globals.\n keys: [question_encoder_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, element_wise_activation_size, category_c123_without_yn_word_to_ix]\n values: [100, 2, 10, 100, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2}]\n\n # Statistics.\n batch_size:\n priority: 0.1\n type: BatchSizeStatistics\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n priority: 0.3\n type: SentenceEmbeddings\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_question_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_encoder_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n priority: 0.7\n type: AccuracyStatistics\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n priority: 1.1\n type: SentenceEmbeddings\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: question_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_encoder_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n priority: 1.3\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n priority: 2.1\n type: GenericImageEncoder\n model: resnet50\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n priority: 3.1\n type: FeedForwardNetwork \n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: image-question fusion #################\n # Element wise multiplication + FF.\n question_image_fusion:\n priority: 4.1\n type: LowRankBilinearPooling\n dropout_rate: 0.5\n streams:\n image_encodings: image_activations\n question_encodings: question_activations\n outputs: element_wise_activations\n globals:\n image_encoding_size: image_encoder_output_size\n question_encoding_size: question_encoder_output_size\n output_size: element_wise_activation_size\n\n question_image_ffn:\n priority: 4.2\n type: FeedForwardNetwork \n hidden_sizes: [100]\n dropout_rate: 0.5\n use_logsoftmax: False\n streams:\n inputs: element_wise_activations\n predictions: question_image_activations\n globals:\n input_size: element_wise_activation_size\n prediction_size: element_wise_activation_size\n\n ################# PIPE 5: image-question-image size fusion #################\n\n # 5th subpipeline: concatenation \n concat:\n priority: 5.1\n type: ConcatenateTensor\n input_streams: [question_image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,10]]\n output_dims: [-1,110]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n ################# PIPE 6: C1 + C2 + C3 questions #################\n\n # Answer encoding for PIPE 6.\n pipe6_c123_answer_indexer:\n priority: 6.1\n type: LabelIndexer\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe6_c123_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n # Sample masking based on categories.\n pipe6_c123_string_to_mask:\n priority: 6.2\n type: StringToMask\n globals:\n word_mappings: category_c123_without_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe6_c123_masks\n\n # Model 4: FFN C123 answering\n pipe6_c123_answer_classifier:\n priority: 6.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe6_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_without_yn\n\n pipe6_c123_nllloss:\n priority: 6.4\n type: NLLLoss\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe6_c123_predictions\n masks: pipe6_c123_masks\n targets: pipe6_c123_answers_ids\n loss: pipe6_c123_loss\n\n pipe6_c123_precision_recall:\n priority: 6.5\n type: PrecisionRecallStatistics\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe6_c123_masks\n predictions: pipe6_c123_predictions\n targets: pipe6_c123_answers_ids\n globals:\n word_mappings: word_mappings_c123_without_yn\n statistics:\n precision: pipe6_c123_precision\n recall: pipe6_c123_recall\n f1score: pipe6_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_c123_prediction_decoder:\n priority: 6.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe6_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n # Viewers.\n viewer:\n priority: 9.3\n type: StreamViewer\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe6_c123_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_resnet50_is_cat_ffn_c123_no_binary_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c123_without_yn_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n dropout_rate: 0.5\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n model: resnet50\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n use_losfotmax: False\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 + C2 + C3 questions #################\n\n # Answer encoding for PIPE 5.\n pipe5_c123_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c123_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n # Sample masking based on categories.\n pipe5_c123_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c123_without_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe5_c123_masks\n\n # Model 4: FFN C1 answering\n pipe5_c123_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c123_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_without_yn\n\n pipe5_c123_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c123_predictions\n masks: pipe5_c123_masks\n targets: pipe5_c123_answers_ids\n loss: pipe5_c123_loss\n\n pipe5_c123_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c123_masks\n predictions: pipe5_c123_predictions\n targets: pipe5_c123_answers_ids\n globals:\n word_mappings: word_mappings_c123_without_yn\n statistics:\n precision: pipe5_c123_precision\n recall: pipe5_c123_recall\n f1score: pipe5_c123_f1score\n\n # C123 Predictions decoder.\n pipe5_prediction_decoder:\n type: WordDecoder\n priority: 5.6\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe5_c123_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe5_c123_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_vgg16_is_cat_ffn_c123_binary_yn_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_c2_c3_binary_yn_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2, \"BINARY\": 3}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n model: vgg16\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 + C2 + C3 + BINARY questions #################\n\n # Answer encoding for PIPE 5.\n pipe5_c123_binary_yn_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c123_binary_yn_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_binary_yn\n word_mappings: word_mappings_c123_binary_yn\n\n # Sample masking based on categories.\n pipe5_c123_binary_yn_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c1_c2_c3_binary_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe5_c123_binary_yn_masks\n\n # Model 4: FFN C1 answering\n pipe5_c123_binary_yn_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c123_binary_yn_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_binary_yn\n\n pipe5_c123_binary_yn_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c123_binary_yn_predictions\n masks: pipe5_c123_binary_yn_masks\n targets: pipe5_c123_binary_yn_answers_ids\n loss: pipe5_c123_binary_yn_loss\n\n pipe5_c123_binary_yn_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c123_binary_yn_masks\n predictions: pipe5_c123_binary_yn_predictions\n targets: pipe5_c123_binary_yn_answers_ids\n globals:\n word_mappings: word_mappings_c123_binary_yn\n statistics:\n precision: pipe5_c123_binary_yn_precision\n recall: pipe5_c123_binary_yn_recall\n f1score: pipe5_c123_binary_yn_f1score\n\n # C123 Predictions decoder.\n pipe5_prediction_decoder:\n type: WordDecoder\n priority: 5.6\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe5_c123_binary_yn_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe5_c123_binary_yn_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_vgg16_is_cat_ffn_c123_no_yn_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.c1_c2_c3_binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_c1_c2_c3_without_yn_word_to_ix]\n values: [100, 2, 10, 100, {\"C1\": 0, \"C2\": 1, \"C3\": 2}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n model: vgg16\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: C1 + C2 + C3 questions #################\n\n # Answer encoding for PIPE 5.\n pipe5_c123_without_yn_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.c1_c2_c3_without_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_c123_without_yn_answers_ids\n globals:\n vocabulary_size: vocabulary_size_c123_without_yn\n word_mappings: word_mappings_c123_without_yn\n\n # Sample masking based on categories.\n pipe5_c123_without_yn_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_c1_c2_c3_without_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe5_c123_without_yn_masks\n\n # Model 4: FFN C1 answering\n pipe5_c123_without_yn_ffn:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_c123_without_yn_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_c123_without_yn\n\n pipe5_c123_without_yn_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_c123_without_yn_predictions\n masks: pipe5_c123_without_yn_masks\n targets: pipe5_c123_without_yn_answers_ids\n loss: pipe5_c123_without_yn_loss\n\n pipe5_c123_without_yn_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_c123_without_yn_masks\n predictions: pipe5_c123_without_yn_predictions\n targets: pipe5_c123_without_yn_answers_ids\n globals:\n word_mappings: word_mappings_c123_without_yn\n statistics:\n precision: pipe5_c123_without_yn_precision\n recall: pipe5_c123_without_yn_recall\n f1score: pipe5_c123_without_yn_f1score\n\n # C123 Predictions decoder.\n pipe5_prediction_decoder:\n type: WordDecoder\n priority: 5.6\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe5_c123_without_yn_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_c123_without_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe5_c123_without_yn_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/vqa_med_2019/vf/lstm_vgg16_is_cat_ffn_only_yn_loss.yml", "content": "# Load config defining tasks for training, validation and testing.\ndefault_configs: vqa_med_2019/default_vqa_med_2019.yml\n\n# Training parameters:\ntraining:\n task:\n categories: C1,C2,C3\n export_sample_weights: ~/data/vqa-med/answers.binary_yn.weights.csv\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n sampler:\n weights: ~/data/vqa-med/answers.binary_yn.weights.csv\n\n# Validation parameters:\nvalidation:\n task:\n categories: C1,C2,C3\n # Appy all preprocessing/data augmentations.\n question_preprocessing: lowercase,remove_punctuation,tokenize\n streams: \n questions: tokenized_questions\n\n\npipeline:\n \n ################# PIPE 0: SHARED #################\n\n # Add global variables.\n global_publisher:\n type: GlobalVariablePublisher\n priority: 0\n # Add input_size to globals.\n keys: [question_lstm_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size, category_binary_yn_word_to_ix]\n values: [100, 2, 10, 100, {\"BINARY\": 3}]\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 0.1\n\n ################# PIPE 0: CATEGORY #################\n\n # Model 1: question embeddings\n pipe0_question_embeddings:\n type: SentenceEmbeddings\n priority: 0.3\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: question_embeddings\n freeze: True\n ###################\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: pipe0_embedded_questions \n \n # Model 2: question RNN\n pipe0_lstm:\n priority: 0.4\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: lstm\n freeze: True\n ###################\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: pipe0_embedded_questions\n predictions: pipe0_questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Model 3: FFN question category\n pipe0_classifier:\n priority: 0.5\n type: FeedForwardNetwork\n # LOAD AND FREEZE #\n load: \n file: ~/image-clef-2019/experiments/q_categorization/20190416_120801/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt\n model: classifier\n freeze: True\n ###################\n hidden: [50]\n streams:\n inputs: pipe0_questions_activations\n predictions: pipe0_predicted_question_categories_preds\n globals:\n input_size: question_lstm_output_size # Set by global publisher\n prediction_size: num_categories # C1,C2,C3,C4, BINARY, UNK\n\n pipe0_category_decoder:\n priority: 0.6\n type: WordDecoder\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe0_predicted_question_categories_preds\n outputs: pipe0_predicted_question_categories_names\n globals:\n vocabulary_size: num_categories\n word_mappings: category_word_mappings\n\n pipe0_category_accuracy:\n type: AccuracyStatistics\n priority: 0.7\n streams:\n targets: category_ids\n predictions: pipe0_predicted_question_categories_preds\n statistics:\n accuracy: categorization_accuracy\n \n ################# PIPE 1: SHARED QUESTION ENCODER #################\n\n # Model 1: question embeddings\n pipe1_question_embeddings:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings_file: glove.6B.50d.txt\n data_folder: ~/data/vqa-med\n word_mappings_file: questions.all.word.mappings.csv\n streams:\n inputs: tokenized_questions\n outputs: embedded_questions \n \n # Model 2: question RNN\n pipe1_lstm:\n priority: 1.2\n type: RecurrentNeuralNetwork\n cell_type: LSTM\n prediction_mode: Last\n initial_state: Trainable\n use_logsoftmax: False\n streams:\n inputs: embedded_questions\n predictions: questions_activations\n globals:\n input_size: embeddings_size\n prediction_size: question_lstm_output_size\n\n # Answer encoding\n pipe1_all_answer_indexer:\n type: LabelIndexer\n priority: 1.3\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: all_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n ################# PIPE 2: SHARED IMAGE ENCODER #################\n\n # Image encoder.\n image_encoder:\n type: GenericImageEncoder\n model: vgg16\n priority: 2.1\n streams:\n inputs: images\n outputs: image_activations\n globals:\n output_size: image_encoder_output_size\n\n ################# PIPE 3: SHARED IMAGE SIZE ENCODER #################\n\n # Model - image size classifier.\n image_size_encoder:\n type: FeedForwardNetwork \n priority: 3.1\n streams:\n inputs: image_sizes\n predictions: image_size_activations\n globals:\n input_size: image_size_encoder_input_size\n prediction_size: image_size_encoder_output_size\n\n ################# PIPE 4: SHARED CONCAT #################\n\n concat:\n type: ConcatenateTensor\n priority: 4.1\n input_streams: [questions_activations,image_activations,image_size_activations]\n # ConcatenateTensor \n dim: 1 # default\n input_dims: [[-1,100],[-1,100],[-1,10]]\n output_dims: [-1,210]\n streams:\n outputs: concatenated_activations\n globals:\n output_size: concatenated_activations_size\n\n\n ################# PIPE 5: Y/N questions #################\n\n # Answer encoding for PIPE 5.\n pipe5_binary_yn_answer_indexer:\n type: LabelIndexer\n priority: 5.1\n data_folder: ~/data/vqa-med\n word_mappings_file: answers.binary_yn.word.mappings.csv\n # Export mappings and size to globals.\n export_word_mappings_to_globals: True\n streams:\n inputs: answers\n outputs: pipe5_binary_yn_answers_ids\n globals:\n vocabulary_size: vocabulary_size_binary_yn\n word_mappings: word_mappings_binary_yn\n\n # Sample masking based on categories.\n pipe5_binary_yn_string_to_mask:\n priority: 5.2\n type: StringToMask\n globals:\n word_mappings: category_binary_yn_word_to_ix\n streams:\n strings: pipe0_predicted_question_categories_names\n string_indices: predicted_c123_by_question_categories_indices # NOT USED\n masks: pipe5_binary_yn_masks\n\n # Model 4: FFN C1 answering\n pipe5_binary_yn_classifier:\n priority: 5.3\n type: FeedForwardNetwork\n hidden: [100]\n dropout_rate: 0.5\n streams:\n inputs: concatenated_activations\n predictions: pipe5_binary_yn_predictions\n globals:\n input_size: concatenated_activations_size\n prediction_size: vocabulary_size_binary_yn\n\n pipe5_binary_yn_nllloss:\n type: NLLLoss\n priority: 5.4\n targets_dim: 1\n use_masking: True\n streams:\n predictions: pipe5_binary_yn_predictions\n masks: pipe5_binary_yn_masks\n targets: pipe5_binary_yn_answers_ids\n loss: pipe5_binary_yn_loss\n\n pipe5_binary_yn_precision_recall:\n type: PrecisionRecallStatistics\n priority: 5.5\n use_word_mappings: True\n use_masking: True\n show_class_scores: True\n #show_confusion_matrix: True\n streams:\n masks: pipe5_binary_yn_masks\n predictions: pipe5_binary_yn_predictions\n targets: pipe5_binary_yn_answers_ids\n globals:\n word_mappings: word_mappings_binary_yn\n statistics:\n precision: pipe5_binary_yn_precision\n recall: pipe5_binary_yn_recall\n f1score: pipe5_binary_yn_f1score\n\n # Y/N Predictions decoder.\n pipe5_prediction_decoder:\n type: WordDecoder\n priority: 5.6\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: pipe5_binary_yn_predictions\n outputs: predicted_answers\n globals:\n word_mappings: word_mappings_binary_yn\n\n ################# PIPE 9: MERGE ANSWERS #################\n\n\n # Viewers.\n viewer:\n type: StreamViewer\n priority: 9.3\n input_streams:\n tokenized_questions, category_names,\n pipe0_predicted_question_categories_names,\n pipe5_binary_yn_masks,\n answers, predicted_answers\n\n\n#: pipeline\n" }, { "path": "configs/wikitext/wikitext_language_modeling_encoder_attndecoder.yml", "content": "# This pipeline applies seq2seq on wikitext-2 to make word-level prediction.\n# It's been made for test purposes only, as it is doing:\n# [word 0 , ... , word 49] -> [word 1 , ... , word 50] (basically copying most of the input)\n#\n# The seq2seq here is implemented throught the use of 2 `RecurrentNeuralNetwork`\n\n# Training parameters:\ntraining:\n task:\n type: &p_type WikiTextLanguageModeling\n data_folder: &data_folder ~/data/language_modeling/wikitext-2\n dataset: &dataset wikitext-2\n subset: train\n sentence_length: 42\n batch_size: 64\n\n # optimizer parameters:\n optimizer:\n type: SGD\n lr: 1.0e-2\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 100\n task:\n type: *p_type\n data_folder: *data_folder\n dataset: *dataset\n subset: valid\n sentence_length: 42\n batch_size: 64\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n dataset: *dataset\n subset: test\n sentence_length: 42\n batch_size: 64\n\npipeline:\n\n # Source encoding - model 1.\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings: glove.6B.50d.txt\n data_folder: *data_folder\n source_vocabulary_files: wiki.train.tokens,wiki.valid.tokens,wiki.test.tokens\n vocabulary_mappings_file: wiki.all.tokenized_words\n additional_tokens: \n export_word_mappings_to_globals: True\n streams:\n inputs: sources\n outputs: embedded_sources\n \n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 2.1\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: targets\n outputs: indexed_targets\n \n # LSTM Encoder\n encoder:\n type: RecurrentNeuralNetwork\n cell_type: GRU\n priority: 3\n initial_state: Trainable\n hidden_size: 50\n num_layers: 1\n use_logsoftmax: False\n output_last_state: True\n prediction_mode: Dense\n ffn_output: False\n streams:\n inputs: embedded_sources\n predictions: s2s_encoder_output\n output_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # LSTM Decoder\n decoder:\n type: AttentionDecoder\n priority: 4\n hidden_size: 50\n num_layers: 1\n use_logsoftmax: False\n autoregression_length: 42\n prediction_mode: Dense\n streams:\n inputs: s2s_encoder_output\n predictions: s2s_decoder_output\n input_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # FF, to resize the from the output size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n streams:\n inputs: s2s_decoder_output\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_targets\n loss: loss\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.1\n # streams:\n # targets: indexed_targets\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n streams:\n targets: indexed_targets\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: sources,targets,indexed_targets,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/wikitext/wikitext_language_modeling_rnn.yml", "content": "# Training parameters:\ntraining:\n task:\n type: &p_type WikiTextLanguageModeling\n data_folder: &data_folder ~/data/language_modeling/wikitext-2\n dataset: &dataset wikitext-2\n subset: train\n sentence_length: 10\n batch_size: 64\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.1\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 10000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 10\n task:\n type: *p_type\n data_folder: *data_folder\n dataset: *dataset\n subset: valid\n sentence_length: 20\n batch_size: 64\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n dataset: *dataset\n subset: test\n sentence_length: 50\n batch_size: 64\n\npipeline:\n\n # Source encoding - model 1.\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1\n embeddings_size: 50\n pretrained_embeddings: glove.6B.50d.txt\n data_folder: *data_folder\n source_vocabulary_files: wiki.train.tokens,wiki.valid.tokens,wiki.test.tokens\n vocabulary_mappings_file: wiki.all.tokenized_words\n additional_tokens: \n export_word_mappings_to_globals: True\n streams:\n inputs: sources\n outputs: embedded_sources\n \n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 2\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: targets\n outputs: indexed_targets\n \n # Model 2: RNN\n lstm:\n type: RecurrentNeuralNetwork\n priority: 3\n initial_state: Zero\n streams:\n inputs: embedded_sources\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size \n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_targets\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.1\n # streams:\n # targets: indexed_targets\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n streams:\n targets: indexed_targets\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: sources,targets,indexed_targets,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/wikitext/wikitext_language_modeling_seq2seq.yml", "content": "# This pipeline applies seq2seq on wikitext-2 to make word-level prediction.\n# It's been made for test purposes only, as it is doing:\n# [word 0 , ... , word 49] -> [word 1 , ... , word 50] (basically copying most of the input)\n#\n# The seq2seq here is implemented throught the use of 2 `RecurrentNeuralNetwork`\n\n# Training parameters:\ntraining:\n task:\n type: &p_type WikiTextLanguageModeling\n data_folder: &data_folder ~/data/language_modeling/wikitext-2\n dataset: &dataset wikitext-2\n subset: train\n sentence_length: 50\n batch_size: 64\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 1.0e-3\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 100\n task:\n type: *p_type\n data_folder: *data_folder\n dataset: *dataset\n subset: valid\n sentence_length: 50\n batch_size: 64\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n dataset: *dataset\n subset: test\n sentence_length: 50\n batch_size: 64\n\npipeline:\n\n # Source encoding - model 1.\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings: glove.6B.50d.txt\n data_folder: *data_folder\n source_vocabulary_files: wiki.train.tokens,wiki.valid.tokens,wiki.test.tokens\n vocabulary_mappings_file: wiki.all.tokenized_words\n additional_tokens: \n export_word_mappings_to_globals: True\n streams:\n inputs: sources\n outputs: embedded_sources\n \n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 2.1\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: targets\n outputs: indexed_targets\n \n # LSTM Encoder\n lstm_encoder:\n type: RecurrentNeuralNetwork\n priority: 3\n initial_state: Trainable\n hidden_size: 300\n num_layers: 3\n use_logsoftmax: False\n output_last_state: True\n prediction_mode: Last\n streams:\n inputs: embedded_sources\n predictions: s2s_encoder_output\n output_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # LSTM Decoder\n lstm_decoder:\n type: RecurrentNeuralNetwork\n priority: 4\n initial_state: Input\n hidden_size: 300\n num_layers: 3\n use_logsoftmax: False\n input_mode: Autoregression_First\n max_autoregression_length: 50\n prediction_mode: Dense\n streams:\n inputs: s2s_encoder_output\n predictions: s2s_decoder_output\n input_state: s2s_state_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # FF, to resize the from the output size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n streams:\n inputs: s2s_decoder_output\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_targets\n loss: loss\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n #accuracy:\n # type: AccuracyStatistics\n # priority: 100.1\n # streams:\n # targets: indexed_targets\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n streams:\n targets: indexed_targets\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: sources,targets,indexed_targets,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/wikitext/wikitext_language_modeling_seq2seq_simple.yml", "content": "# This pipeline applies seq2seq on wikitext-2 to make word-level prediction.\n# It's been made for test purposes only, as it is doing:\n# [word 0 , ... , word 49] -> [word 1 , ... , word 50] (basically copying most of the input)\n#\n# The seq2seq here is implemented throught the use of a simplified seq2seq component `Seq2Seq`\n\n# Training parameters:\ntraining:\n task:\n type: &p_type WikiTextLanguageModeling\n data_folder: &data_folder ~/data/language_modeling/wikitext-2\n dataset: &dataset wikitext-2\n subset: train\n sentence_length: 50\n batch_size: 64\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 1.0e-3\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 1000000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 100\n task:\n type: *p_type\n data_folder: *data_folder\n dataset: *dataset\n subset: valid\n sentence_length: 50\n batch_size: 64\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n dataset: *dataset\n subset: test\n sentence_length: 50\n batch_size: 64\n\npipeline:\n\n # Source encoding - model 1.\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1.1\n embeddings_size: 50\n pretrained_embeddings: glove.6B.50d.txt\n data_folder: *data_folder\n source_vocabulary_files: wiki.train.tokens,wiki.valid.tokens,wiki.test.tokens\n vocabulary_mappings_file: wiki.all.tokenized_words\n additional_tokens: \n export_word_mappings_to_globals: True\n streams:\n inputs: sources\n outputs: embedded_sources\n \n # Target encoding.\n target_indexer:\n type: SentenceIndexer\n priority: 2.1\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: targets\n outputs: indexed_targets\n \n # LSTM seq2seq\n lstm_encoder:\n type: Seq2Seq\n priority: 3\n initial_state: Trainable\n hidden_size: 300\n num_layers: 3\n use_logsoftmax: False\n streams:\n inputs: embedded_sources\n predictions: s2s_output\n globals:\n input_size: embeddings_size\n prediction_size: embeddings_size \n\n # FF, to resize the from the hidden size of the seq2seq to the size of the target vector\n ff_resize_s2s_output:\n type: FeedForwardNetwork \n use_logsoftmax: True\n dimensions: 3\n priority: 5\n streams:\n inputs: s2s_output\n globals:\n input_size: embeddings_size\n prediction_size: vocabulary_size\n\n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n num_targets_dims: 2\n streams:\n targets: indexed_targets\n loss: loss\n\n # Prediction decoding.\n prediction_decoder:\n type: SentenceIndexer\n priority: 10\n # Reverse mode.\n reverse: True\n # Use distributions as inputs.\n use_input_distributions: True\n data_folder: *data_folder\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: prediction_sentences\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 100.0\n\n bleu:\n type: BLEUStatistics\n priority: 100.2\n streams:\n targets: indexed_targets\n\n \n # Viewers.\n viewer:\n type: StreamViewer\n priority: 100.3\n input_streams: sources,targets,indexed_targets,prediction_sentences\n\n#: pipeline\n" }, { "path": "configs/wily/dummy_language_identification_bow.yml", "content": "# Training parameters:\ntraining:\n task:\n type: &p_type DummyLanguageIdentification\n batch_size: 2\n use_train_data: True\n streams: &p_streams\n inputs: sentences\n targets: languages\n\n # optimizer parameters:\n optimizer:\n type: SGD\n lr: 0.1\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 10000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 10\n task:\n type: *p_type\n batch_size: 2\n use_train_data: True\n streams: *p_streams\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n batch_size: 2\n use_train_data: False\n streams: *p_streams\n\npipeline:\n #load: /users/tomaszkornuta/experiments/dummylanguageidentification/language_classifier/20190301_145416/checkpoints/language_classifier_best.pt\n #freeze: True\n #disable: prediction_decoder,accuracy\n # Sentences encoding.\n sentence_tokenizer:\n type: SentenceTokenizer\n priority: 1\n streams: \n inputs: sentences\n outputs: tokenized_sentences\n\n sentence_encoder:\n type: SentenceOneHotEncoder\n priority: 2\n data_folder: ~/data/language_identification/dummy\n source_vocabulary_files: x_training.txt,x_test.txt\n word_mappings_file: word_encodings.csv\n streams:\n inputs: tokenized_sentences\n outputs: encoded_sentences\n globals:\n vocabulary_size: sentence_vocab_size \n\n bow_encoder:\n type: BOWEncoder\n priority: 3\n streams:\n inputs: encoded_sentences\n outputs: bow_sencentes\n globals:\n bow_size: sentence_vocab_size # Set by sentence_encoder.\n \n # Targets encoding.\n label_indexer:\n type: LabelIndexer\n priority: 4\n data_folder: ~/data/language_identification/dummy\n source_vocabulary_files: y_training.txt,y_test.txt\n word_mappings_file: language_name_encodings.csv\n # Export word mappings, so other can reuse them.\n export_word_mappings_to_globals: True\n streams:\n inputs: languages\n outputs: indexed_languages\n globals:\n vocabulary_size: label_vocab_size \n word_mappings: label_word_mappings\n\n # Model\n classifier:\n type: FeedForwardNetwork \n #freeze: True\n priority: 5\n streams:\n inputs: bow_sencentes\n globals:\n input_size: sentence_vocab_size # Set by sentence_encoder.\n prediction_size: label_vocab_size\n \n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n streams:\n targets: indexed_languages\n #predictions: encoded_predictions\n loss: loss\n \n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 8\n # Use the same word mappings as label indexer.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_labels\n globals:\n vocabulary_size: label_vocab_size\n word_mappings: label_word_mappings\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 9\n accuracy:\n type: AccuracyStatistics\n priority: 10\n streams:\n targets: indexed_languages\n\n #: pipeline\n" }, { "path": "configs/wily/wily_language_identification_bow.yml", "content": " # Training parameters:\ntraining:\n task:\n type: &p_type WiLYLanguageIdentification\n data_folder: &data_folder '~/data/language_identification/wily'\n batch_size: 64\n streams: &p_streams\n inputs: sentences\n targets: languages\n\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [0, 117000]\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.1\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 10000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 10\n task:\n type: *p_type\n data_folder: *data_folder\n batch_size: 64\n streams: *p_streams\n\n # Use sampler that operates on a subset.\n sampler:\n type: SubsetRandomSampler\n indices: [117000, 117500]\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n batch_size: 64\n use_train_data: False\n streams: *p_streams\n\npipeline:\n\n #load: /users/tomaszkornuta/experiments/dummylanguageidentification/language_classifier/20190301_145416/checkpoints/language_classifier_best.pt\n #freeze: True\n #disable: prediction_decoder,accuracy\n # Sentences encoding.\n sentence_tokenizer:\n type: SentenceTokenizer\n priority: 1\n streams: \n inputs: sentences\n outputs: tokenized_sentences\n\n sentence_encoder:\n type: SentenceOneHotEncoder\n priority: 2\n data_folder: ~/data/language_identification/wily\n source_vocabulary_files: x_train.txt,x_test.txt\n word_mappings_file: word_encodings.csv\n streams:\n inputs: tokenized_sentences\n outputs: encoded_sentences\n globals:\n vocabulary_size: sentence_vocab_size \n word_mappings: sentence_word_mappings\n \n bow_encoder:\n type: BOWEncoder\n priority: 3\n streams:\n inputs: encoded_sentences\n outputs: bow_sencentes\n globals:\n bow_size: sentence_vocab_size # Set by sentence_encoder.\n \n # Targets encoding.\n label_indexer:\n type: LabelIndexer\n priority: 4\n data_folder: ~/data/language_identification/wily\n source_vocabulary_files: y_train.txt,y_test.txt\n word_mappings_file: language_name_encodings.csv\n streams:\n inputs: languages\n outputs: indexed_languages\n globals:\n vocabulary_size: label_vocab_size \n word_mappings: label_word_mappings\n\n # Model\n classifier:\n type: FeedForwardNetwork \n #freeze: True\n priority: 5\n streams:\n inputs: bow_sencentes\n globals:\n input_size: sentence_vocab_size # Set by sentence_encoder.\n prediction_size: label_vocab_size # Set by target_encoder.\n \n # Loss\n nllloss:\n type: NLLLoss\n priority: 6\n streams:\n targets: indexed_languages\n #predictions: encoded_predictions\n loss: loss\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 8\n data_folder: ~/data/language_identification/wily\n word_mappings_file: language_name_encodings.csv\n streams:\n inputs: predictions\n outputs: predicted_labels\n globals:\n vocabulary_size: label_vocab_size \n word_mappings: label_word_mappings\n\n # Statistics.\n batch_size:\n type: BatchSizeStatistics\n priority: 9\n accuracy:\n type: AccuracyStatistics\n priority: 10\n streams:\n targets: indexed_languages\n\n #: pipeline\n" }, { "path": "configs/wily/wily_ngram_language_modeling.yml", "content": " # Training parameters:\ntraining:\n task:\n type: &p_type WiLYNGramLanguageModeling\n data_folder: &data_folder '~/data/language_identification/wily'\n context: &context 2\n batch_size: 64\n use_train_data: True\n streams: &p_streams\n inputs: ngrams\n #targets: targets\n\n # Use sampler that operates on a subset.\n #sampler:\n # type: SubsetRandomSampler\n # indices: [0, 117000]\n\n # optimizer parameters:\n optimizer:\n type: Adam\n lr: 0.1\n\n # settings parameters\n terminal_conditions:\n loss_stop_threshold: 1.0e-2\n episode_limit: 10000\n epoch_limit: 100\n\n# Validation parameters:\nvalidation:\n partial_validation_interval: 10\n task:\n type: *p_type\n data_folder: *data_folder\n context: *context\n batch_size: 64\n use_train_data: True\n streams: *p_streams\n\n # Use sampler that operates on a subset.\n #sampler:\n # type: SubsetRandomSampler\n # indices: [117000, 117500]\n\n# Testing parameters:\ntest:\n task:\n type: *p_type \n data_folder: *data_folder\n context: *context\n batch_size: 64\n use_train_data: False\n streams: *p_streams\n\npipeline:\n #load: /users/tomaszkornuta/experiments/dummylanguageidentification/language_classifier/20190301_145416/checkpoints/language_classifier_best.pt\n #freeze: True\n #disable: prediction_decoder,accuracy\n\n # Source encoding.\n sentence_tokenizer:\n type: SentenceTokenizer\n priority: 1.1\n streams: \n inputs: ngrams\n outputs: tokenized_ngrams\n\n source_sentence_embedding:\n type: SentenceEmbeddings\n priority: 1.2\n embeddings_size: 50\n data_folder: *data_folder\n source_vocabulary_files: x_training.txt,x_test.txt\n word_mappings_file: word_encodings.csv\n additional_tokens: \n # Export word mappings \n export_word_mappings_to_globals: True\n streams:\n inputs: tokenized_ngrams\n outputs: embedded_ngrams\n globals:\n vocabulary_size: sentence_vocab_size \n \n\n # Targets encoding.\n target_indexer:\n type: LabelIndexer\n priority: 2\n data_folder: *data_folder\n # Use mappings created for source sentence embeddings.\n import_word_mappings_from_globals: True\n streams:\n inputs: targets\n outputs: indexed_targets\n \n # Reshapes tensors.\n reshaper:\n type: ReshapeTensor\n input_dims: [-1, -1, 50]\n output_dims: [-1, 100]\n priority: 5\n streams:\n inputs: embedded_ngrams\n outputs: reshaped_ngrams\n globals:\n output_size: reshaped_ngrams_size\n\n # Model 2: classifier.\n classifier:\n type: FeedForwardNetwork \n priority: 6\n hidden_sizes: [100]\n streams:\n inputs: reshaped_ngrams\n globals:\n input_size: reshaped_ngrams_size\n prediction_size: sentence_vocab_size\n\n # Loss.\n nllloss:\n type: NLLLoss\n priority: 7\n streams:\n targets: indexed_targets\n predictions: predictions\n loss: loss\n\n # Predictions decoder.\n prediction_decoder:\n type: WordDecoder\n priority: 8\n data_folder: *data_folder\n # Use mappings from source embeddings.\n import_word_mappings_from_globals: True\n streams:\n inputs: predictions\n outputs: predicted_labels\n\n # Statistics.\n accuracy:\n type: AccuracyStatistics\n priority: 10\n streams:\n targets: indexed_targets\n\n #: pipeline\n" }, { "path": "ptp/__init__.py", "content": "from .utils import *\n\nfrom .application import *\n\n# Components.\nfrom .components.component import Component\n\nfrom .components.language import *\n\nfrom .components.losses import *\n\nfrom .components.masking import *\n\nfrom .components.models import *\n\nfrom .components.tasks.task import Task\nfrom .components.tasks.image_text_to_class import *\nfrom .components.tasks.image_to_class import *\nfrom .components.tasks.text_to_class import *\nfrom .components.tasks.text_to_text import *\n\nfrom .components.publishers import *\n\nfrom .components.transforms import *\n\nfrom .components.statistics import *\n\nfrom .components.viewers import *\n\n# Rest.\nfrom .configuration import *\nfrom .data_types import *\n\n# Workers.\nfrom .workers import *\n\n" }, { "path": "ptp/application/__init__.py", "content": "from .component_factory import ComponentFactory\nfrom .pipeline_manager import PipelineManager\nfrom .task_manager import TaskManager\nfrom .sampler_factory import SamplerFactory\n\n__all__ = [\n 'ComponentFactory',\n 'PipelineManager',\n 'TaskManager',\n 'SamplerFactory',\n ]\n" }, { "path": "ptp/application/component_factory.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport inspect\n\nimport ptp\n\nfrom ptp.configuration.configuration_error import ConfigurationError\n\n\nclass ComponentFactory(object):\n \"\"\"\n Class instantiating the components using the passed config.\n \"\"\"\n\n @staticmethod\n def check_inheritance(class_obj, parent_class_name):\n \"\"\"\n Checks whether given class inherits (even indirectly) from parent class.\n \"\"\"\n # Check if class is derived (even indirectly) from Component.\n for c in inspect.getmro(class_obj):\n if c.__name__ == parent_class_name:\n return True\n return False\n\n\n\n @staticmethod\n def build(name, config):\n \"\"\"\n Method creates a single component on the basis of configuration section.\n Raises ConfigurationError exception when encountered issues.\n\n :param name: Name of the section/component.\n\n :param config: Parameters used to instantiate all components.\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n :return: tuple (component, component class).\n \"\"\"\n\n # Check presence of type.\n if 'type' not in config:\n raise ConfigurationError(\"Section {} does not contain the key 'type' defining the component type\".format(name))\n\n # Get the class type.\n c_type = config[\"type\"]\n\n # Get class object.\n if c_type.find(\"ptp.\") != -1:\n # Try to evaluate it directly.\n class_obj = eval(c_type)\n else:\n try:\n # Try to find it in the main \"ptp\" namespace.\n class_obj = getattr(ptp, c_type)\n except AttributeError:\n raise ConfigurationError(\"Class '{}' not found in the list of Component classes\".format(c_type))\n\n # Check if class is derived (even indirectly) from Component.\n if not ComponentFactory.check_inheritance(class_obj, ptp.Component.__name__):\n raise ConfigurationError(\"Class '{}' is not derived from the Component class\".format(c_type))\n\n # Instantiate component.\n component = class_obj(name, config)\n\n return component, class_obj\n" }, { "path": "ptp/application/pipeline_manager.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport os\nimport torch\nfrom datetime import datetime\nfrom numpy import inf,average\n\nimport ptp.components\n\nimport ptp.utils.logger as logging\nfrom ptp.utils.app_state import AppState\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.application.component_factory import ComponentFactory\nfrom ptp.utils.data_streams_parallel import DataStreamsParallel\n\n\ncomponents_to_skip_in_data_parallel = [\"SentenceEmbeddings\", \"IndexEmbeddings\"]\n\n\nclass PipelineManager(object):\n \"\"\"\n Class responsible for instantiating the pipeline consisting of several components.\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the pipeline manager.\n\n :param config: Parameters used to instantiate all required components.\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Initialize the logger.\n self.name = name\n self.config = config\n self.app_state = AppState()\n # Initialize logger.\n self.logger = logging.initialize_logger(self.name) \n\n # Set initial values of all pipeline elements.\n # Empty list of all components, sorted by their priorities.\n self.__components = {}\n # Empty list of all models - it will contain only \"references\" to objects stored in the components list.\n self.models = []\n # Empty list of all losses - it will contain only \"references\" to objects stored in the components list.\n self.losses = []\n\n # Initialization of best loss - as INF.\n self.best_loss = inf\n self.best_status = \"Unknown\"\n # Indicates the last time when the validation loss went down.\n # 0 means currntly, 1 means during previous validation etc.\n self.validation_loss_down_counter = 0\n\n\n def build(self, use_logger=True):\n \"\"\"\n Method creating the pipeline, consisting of:\n - a list components ordered by the priority (dictionary).\n - task (as a separate \"link\" to object in the list of components, instance of a class derrived from Task class)\n - models (separate list with link to objects in components dict)\n - losses (selarate list with links to objects in components dict)\n\n :param use_logger: Logs the detected errors (DEFAULT: True)\n\n :return: number of detected errors.\n \"\"\"\n errors = 0\n self.__priorities = []\n\n # Special section names to \"skip\".\n sections_to_skip = \"name load freeze disable\".split()\n disabled_components = ''\n # Add components to disable by the ones from configuration file.\n if \"disable\" in self.config:\n disabled_components = [*disabled_components, *self.config[\"disable\"].replace(\" \",\"\").split(\",\")]\n # Add components to disable by the ones from command line arguments.\n if (self.app_state.args is not None) and (self.app_state.args.disable != ''):\n disabled_components = [*disabled_components, *self.app_state.args.disable.split(\",\")]\n\n # Organize all components according to their priorities.\n for c_key, c_config in self.config.items():\n\n try:\n # Skip \"special\" pipeline sections.\n if c_key in sections_to_skip:\n #self.logger.info(\"Skipping section '{}'\".format(c_key))\n continue\n # Skip \"disabled\" components.\n if c_key in disabled_components:\n self.logger.info(\"Disabling component '{}'\".format(c_key))\n continue\n\n # Check presence of priority.\n if 'priority' not in c_config:\n raise KeyError(\"Section '{}' does not contain the key 'priority' defining the pipeline order\".format(c_key))\n\n # Get the priority.\n try:\n c_priority = float(c_config[\"priority\"])\n except ValueError:\n raise ConfigurationError(\"Priority [{}] in section '{}' is not a floating point number\".format(c_config[\"priority\"], c_key))\n\n # Check uniqueness of the priority.\n if c_priority in self.__components.keys():\n raise ConfigurationError(\"Found more than one component with the same priority [{}]\".format(c_priority))\n\n # Ok, got the component name with priority. Save it.\n # Later we will \"plug\" the adequate component in this place.\n self.__components[c_priority] = c_key\n\n except ConfigurationError as e:\n if use_logger:\n self.logger.error(e)\n errors += 1\n continue\n except KeyError as e:\n if use_logger:\n self.logger.error(e)\n errors += 1\n continue\n # end try/else\n # end for\n\n if use_logger:\n self.logger.info(\"Building pipeline with {} components\".format(len(self.__components)))\n\n # Do not continue if found errors.\n if errors > 0:\n return errors\n\n # Sort priorities.\n self.__priorities=sorted(self.__components.keys()) \n\n for c_priority in self.__priorities:\n try:\n # The section \"key\" will be used as \"component\" name.\n c_key = self.__components[c_priority]\n # Get section.\n c_config = self.config[c_key]\n \n if use_logger:\n self.logger.info(\"Creating component '{}' ({}) with priority [{}]\".format(c_key, c_config[\"type\"], c_priority))\n\n # Create component.\n component, class_obj = ComponentFactory.build(c_key, c_config)\n\n # Check if class is derived (even indirectly) from Task.\n if ComponentFactory.check_inheritance(class_obj, ptp.Task.__name__):\n raise ConfigurationError(\"Object '{}' cannot be instantiated as part of pipeline, \\\n as its class type '{}' is derived from Task class!\".format(c_key, class_obj.__name__))\n\n # Add it to dict.\n self.__components[c_priority] = component\n\n # Check if class is derived (even indirectly) from Model.\n if ComponentFactory.check_inheritance(class_obj, ptp.Model.__name__):\n # Add to list.\n self.models.append(component)\n\n # Check if class is derived (even indirectly) from Loss.\n if ComponentFactory.check_inheritance(class_obj, ptp.Loss.__name__):\n # Add to list.\n self.losses.append(component)\n\n except ConfigurationError as e:\n if use_logger:\n self.logger.error(\"Detected configuration error while creating the component '{}' instance:\\n {}\".format(c_key, e))\n errors += 1\n continue\n except KeyError as e:\n if use_logger:\n self.logger.error(\"Detected key error while creating the component '{}' instance: required key '{}' is missing\".format(c_key, e))\n errors += 1\n continue\n # end try/else\n # end for\n\n # Return detected errors.\n return errors\n\n\n def save(self, chkpt_dir, training_status, loss):\n \"\"\"\n Generic method saving the parameters of all models in the pipeline to a file.\n\n :param chkpt_dir: Directory where the model will be saved.\n :type chkpt_dir: str\n\n :param training_status: String representing the current status of training.\n :type training_status: str\n\n :return: True if this is currently the best model (until the current episode, considering the loss).\n \"\"\"\n # Checkpoint to be saved.\n chkpt = {'name': self.name,\n 'timestamp': datetime.now(),\n 'episode': self.app_state.episode,\n 'loss': loss,\n 'status': training_status,\n 'status_timestamp': datetime.now(),\n }\n \n model_str = ''\n # Save state dicts of all models.\n for model in self.models:\n # Check if model is wrapped in dataparallel.\n if (type(model).__name__ == \"DataStreamsParallel\"):\n model.module.save_to_checkpoint(chkpt)\n model_str += \" + Model '{}' [{}] params saved \\n\".format(model.module.name, type(model.module).__name__)\n else:\n model.save_to_checkpoint(chkpt)\n model_str += \" + Model '{}' [{}] params saved \\n\".format(model.name, type(model).__name__)\n\n # Save the intermediate checkpoint.\n if self.app_state.args.save_intermediate:\n filename = chkpt_dir + self.name + '_episode_{:05d}.pt'.format(self.app_state.episode)\n torch.save(chkpt, filename)\n log_str = \"Exporting pipeline '{}' parameters to checkpoint:\\n {}\\n\".format(self.name, filename)\n log_str += model_str\n self.logger.info(log_str)\n\n # Save the best \"model\".\n # loss = loss.cpu() # moving loss value to cpu type to allow (initial) comparison with numpy type\n if loss < self.best_loss:\n # Save best loss and status.\n self.best_loss = loss\n self.best_status = training_status\n # Save checkpoint.\n filename = chkpt_dir + self.name + '_best.pt'\n torch.save(chkpt, filename)\n log_str = \"Exporting pipeline '{}' parameters to checkpoint:\\n {}\\n\".format(self.name, filename)\n log_str += model_str\n self.logger.info(log_str)\n # Ok, loss went down, reset the counter.\n self.validation_loss_down_counter = 0\n return True\n elif self.best_status != training_status:\n filename = chkpt_dir + self.name + '_best.pt'\n # Load checkpoint.\n chkpt_loaded = torch.load(filename, map_location=lambda storage, loc: storage)\n # Update status and status time.\n chkpt_loaded['status'] = training_status\n chkpt_loaded['status_timestamp'] = datetime.now()\n # Save updated checkpoint.\n torch.save(chkpt_loaded, filename)\n self.logger.info(\"Updated training status in checkpoint:\\n {}\".format(filename))\n # Else: that was not the best \"model\".\n # Loss didn't went down, increment the counter.\n self.validation_loss_down_counter += 1\n return False\n\n def load(self, checkpoint_file):\n \"\"\"\n Loads parameters of models in the pipeline from the specified checkpoint file.\n\n :param checkpoint_file: File containing dictionary with states of all models in the pipeline with some additional checkpoint statistics.\n\n \"\"\"\n # Load checkpoint\n checkpoint_file = os.path.expanduser(checkpoint_file.replace(\" \",\"\"))\n # This is to be able to load a CUDA-trained model on CPU\n chkpt = torch.load(checkpoint_file, map_location=lambda storage, loc: storage)\n\n log_str = \"Loading models constituting the '{}' pipeline from checkpoint defined in {} (episode: {}, loss: {}, status: {}):\\n\".format(\n chkpt['name'],\n chkpt['timestamp'],\n chkpt['episode'],\n chkpt['loss'],\n chkpt['status']\n )\n model_str = ''\n warning = False\n # Save state dicts of all models.\n for model in self.models:\n try:\n # Load model.\n model.load_from_checkpoint(chkpt)\n model_str += \" + Model '{}' [{}] params loaded\\n\".format(model.name, type(model).__name__)\n except KeyError:\n model_str += \" + Model '{}' [{}] params not found in checkpoint!\\n\".format(model.name, type(model).__name__)\n warning = True\n\n # Log results.\n log_str += model_str\n if warning:\n self.logger.warning(log_str)\n else:\n self.logger.info(log_str)\n\n def load_models(self):\n \"\"\"\n Method analyses the configuration and loads models one by one by looking whether they got 'load' variable present in their configuration section.\n\n ..note::\n The 'load' variable should contain path with filename of the checkpoint from which we want to load particular model.\n \"\"\"\n error = False\n log_str = ''\n # Iterate over models.\n for model in self.models:\n if \"load\" in model.config.keys():\n try:\n # Determine whether checkpoint is a string (filename) or list.\n checkpoint = model.config[\"load\"]\n if type(checkpoint) == str:\n checkpoint_filename = checkpoint\n checkpoint_model = None\n else: # Assume dictionary.\n if 'file' not in checkpoint.keys() or 'model' not in checkpoint.keys():\n log_str += \" + The 'load' section of model '{}' is incorrect: it must contain a single string (with checkpoint filename) or a dictionary (with two sections: checkpoint 'file' and 'model' to load)\\n\".format(\n model.name\n )\n error = True\n continue\n # Ok!\n checkpoint_filename = checkpoint[\"file\"]\n checkpoint_model = checkpoint[\"model\"]\n\n # Check if file exists. \n checkpoint_filename = os.path.expanduser(checkpoint_filename.replace(\" \",\"\"))\n if not os.path.isfile(checkpoint_filename):\n log_str += \" + Could not import parameters of model '{}' from checkpoint '{}' as file does not exist\\n\".format(\n model.name,\n checkpoint_filename\n )\n error = True\n continue\n\n # Load checkpoint.\n # This is to be able to load a CUDA-trained model on CPU\n chkpt = torch.load(checkpoint_filename, map_location=lambda storage, loc: storage)\n\n log_str += \" + Importing model '{}' from pipeline '{}' parameters from checkpoint from {} (episode: {}, loss: {}, status: {})\\n\".format(\n model.name,\n chkpt['name'],\n chkpt['timestamp'],\n chkpt['episode'],\n chkpt['loss'],\n chkpt['status']\n )\n # Load model.\n model.load_from_checkpoint(chkpt, checkpoint_model)\n\n log_str += \" + Model '{}' [{}] params loaded\\n\".format(model.name, type(model).__name__)\n except KeyError:\n log_str += \" + Model '{}' [{}] params not found in checkpoint!\\n\".format(model.name, type(model).__name__)\n error = True\n\n # Log results.\n if error:\n # Log errors - always.\n log_str = 'Failed while trying to load the pre-trained models:\\n' + log_str\n self.logger.error(log_str)\n # Exit by following the logic: if user wanted to load the model but failed, then continuing the experiment makes no sense.\n exit(-6)\n else:\n # Log info - only if some models were loaded.\n if len(log_str) > 0:\n log_str = 'Successfully loaded the pre-trained models:\\n' + log_str\n self.logger.info(log_str)\n\n\n def freeze_models(self):\n \"\"\"\n Method analyses the configuration and freezes:\n - all models when 'freeze' flag for whoe pipeline is set,\n - individual models when their 'freeze' flags are set.\n \"\"\"\n # Check freeze all option.\n if \"freeze\" in self.config.keys():\n freeze_all = bool(self.config[\"freeze\"])\n else: \n freeze_all = False\n \n # Iterate over models.\n for model in self.models:\n if \"freeze\" in model.config.keys():\n if bool(model.config[\"freeze\"]):\n model.freeze()\n elif freeze_all:\n model.freeze()\n \n\n def __getitem__(self, number):\n \"\"\"\n Returns the component, using the enumeration resulting from priorities.\n\n :param number: Number of the component in the pipeline.\n :type key: str\n\n :return: object of type :py:class:`Component`.\n\n \"\"\"\n return self.__components[self.__priorities[number]]\n\n\n def __len__(self):\n \"\"\"\n Returns the number of objects in the pipeline (excluding tasks)\n :return: Length of the :py:class:`Pipeline`.\n\n \"\"\"\n length = len(self.__priorities) \n return length\n\n\n def summarize_all_components_header(self):\n \"\"\"\n Creates the summary header containing components with inputs-outputs definitions.\n\n :return: Summary header as a str.\n \"\"\"\n summary_str = 'Summary of the created pipeline:\\n'\n summary_str += '='*80 + '\\n'\n summary_str += 'Pipeline\\n'\n summary_str += ' + Component name (type) [priority]\\n'\n summary_str += ' Inputs:\\n' \n summary_str += ' key: dims, types, description\\n'\n summary_str += ' Outputs:\\n' \n summary_str += ' key: dims, types, description\\n'\n summary_str += '=' * 80 + '\\n'\n return summary_str\n\n\n def summarize_all_components(self):\n \"\"\"\n Summarizes the pipeline by showing all its components (excluding task).\n\n :return: Summary as a str.\n \"\"\"\n summary_str = '' \n for prio in self.__priorities:\n # Get component\n comp = self.__components[prio]\n if type(comp) == str:\n summary_str += ' + {} (None: not created) [{}]\\n'.format(comp, prio)\n else:\n summary_str += comp.summarize_io(prio)\n summary_str += '=' * 80 + '\\n'\n return summary_str\n\n def summarize_models_header(self):\n \"\"\"\n Creates the summary header containing details of models.\n\n :return: Summary header as a str.\n \"\"\"\n summary_str = 'Summary of the models in the pipeline:\\n'\n summary_str += '='*80 + '\\n'\n summary_str += 'Model name (Type) \\n'\n summary_str += ' + Submodule name (Type) \\n'\n summary_str += ' Matrices: [(name, dims), ...]\\n'\n summary_str += ' Trainable Params: #\\n'\n summary_str += ' Non-trainable Params: #\\n'\n summary_str += '=' * 80 + '\\n'\n return summary_str\n\n def summarize_models(self):\n \"\"\"\n Summarizes the pipeline by showing all its components (excluding task).\n\n :return: Summary as a str.\n \"\"\"\n summary_str = '' \n for model in self.models:\n summary_str += model.summarize()\n return summary_str\n\n\n def handshake(self, data_streams, log=True):\n \"\"\"\n Performs handshaking of inputs and outputs definitions of all components in the pipeline.\n\n :param data_streams: Initial datadict returned by the task.\n\n :param log: Logs the detected errors and info (DEFAULT: True)\n\n :return: Number of detected errors.\n \"\"\"\n errors = 0\n\n for prio in self.__priorities:\n # Get component\n comp = self.__components[prio]\n # Handshake inputs and outputs.\n errors += comp.handshake_input_definitions(data_streams, log)\n errors += comp.export_output_definitions(data_streams, log)\n\n # Log final definition.\n if errors == 0 and log:\n self.logger.info(\"Handshake successfull\")\n def_str = \"Final definition of DataStreams used in pipeline:\\n\"\n def_str += '='*80 + '\\n'\n for item in data_streams.items():\n def_str += ' {}\\n'.format(item)\n def_str += '='*80 + '\\n'\n self.logger.info(def_str)\n\n return errors\n\n\n def forward(self, data_streams):\n \"\"\"\n Method responsible for processing the data dict, using all components in the components queue.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing both input data to be processed and that will be extended by the results.\n\n \"\"\"\n if self.app_state.args.use_gpu:\n data_streams.to(device = self.app_state.device)\n\n for prio in self.__priorities:\n # Get component\n comp = self.__components[prio]\n if (type(comp).__name__ == \"DataStreamsParallel\"):\n # Forward of wrapper returns outputs in separate DataStreams.\n outputs = comp(data_streams)\n # Postprocessing: copy only the outputs of the wrapped model.\n for key in comp.module.output_data_definitions().keys():\n data_streams.publish({key: outputs[key]})\n else: \n # \"Normal\" forward step.\n comp(data_streams)\n # Move data to device.\n data_streams.to(device = self.app_state.device)\n\n\n def eval(self):\n \"\"\" \n Sets evaluation mode for all models in the pipeline.\n \"\"\"\n for model in self.models:\n model.eval\n\n\n def train(self):\n \"\"\" \n Sets evaluation mode for all models in the pipeline.\n \"\"\"\n for model in self.models:\n model.train()\n\n\n def cuda(self):\n \"\"\" \n Moves all models to GPU.\n \"\"\"\n self.logger.info(\"Moving model(s) to GPU(s)\")\n if self.app_state.use_dataparallel:\n self.logger.info(\"Using data parallelization on {} GPUs!\".format(torch.cuda.device_count()))\n\n # Regenerate the model list AND overwrite the models on the list of components.\n self.models = []\n for key, component in self.__components.items():\n\n # Check if class is derived (even indirectly) from Model.\n if ComponentFactory.check_inheritance(type(component), ptp.Model.__name__):\n model = component\n # Wrap model with DataStreamsParallel when required.\n if self.app_state.use_dataparallel and type(model).__name__ not in components_to_skip_in_data_parallel:\n print(\"Moving to GPU\", model.name)\n model = DataStreamsParallel(model)\n # Mode to cuda.\n model.to(self.app_state.device)\n\n # Add to list.\n self.models.append(model)\n # \"Overwrite\" model on the component list.\n self.__components[key] = model\n\n def zero_grad(self):\n \"\"\" \n Resets gradients in all trainable components of the pipeline.\n \"\"\"\n for model in self.models:\n model.zero_grad()\n\n\n def backward(self, data_streams):\n \"\"\"\n Propagates gradients backwards, starting from losses returned by every loss component in the pipeline.\n If using many losses the components derived from loss must overwrite the ''loss_keys()'' method.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing both input data to be processed and that will be extended by the results.\n\n \"\"\"\n if (len(self.losses) == 0):\n raise ConfigurationError(\"Cannot train using backpropagation as there are no 'Loss' components\")\n # Calculate total number of backward passes.\n total_passes = sum([len(loss.loss_keys()) for loss in self.losses])\n\n # All but the last call to backward should have the retain_graph=True option.\n pass_counter = 0\n for loss in self.losses:\n for key in loss.loss_keys():\n pass_counter += 1\n if pass_counter == total_passes:\n # Last pass.\n data_streams[key].backward()\n else:\n # \"Other pass.\"\n data_streams[key].backward(retain_graph=True)\n\n\n def return_loss_on_batch(self, stat_col):\n \"\"\"\n Sums all losses and returns a single value that can be used e.g. in terminal condition or model(s) saving.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing both input data to be processed and that will be extended by the results.\n\n :return: Loss (scalar value).\n \"\"\"\n return stat_col[\"total_loss\"][-1]\n\n\n def return_loss_on_set(self, stat_agg):\n \"\"\"\n Sums all losses and returns a single value that can be used e.g. in terminal condition or model(s) saving.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing both input data to be processed and that will be extended by the results.\n\n :return: Loss (scalar value).\n \"\"\"\n\n return stat_agg[\"total_loss\"]\n\n\n def parameters(self, recurse=True):\n \"\"\"\n Returns an iterator over parameters of all trainable components.\n\n This is typically passed to an optimizer.\n\n Args:\n recurse (bool): if True, then yields parameters of this module\n and all submodules. Otherwise, yields only parameters that\n are direct members of this module.\n\n Yields:\n Parameter: module parameter\n\n Example::\n\n \"\"\"\n for model in self.models:\n for _, param in model.named_parameters(recurse=recurse):\n yield param\n\n\n def named_parameters(self, recurse=True):\n \"\"\"\n Returns an iterator over all named parameters of all trainable components.\n \"\"\"\n for model in self.models:\n for name, param in model.named_parameters(recurse=recurse):\n yield name, param\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds statistics for every component in the pipeline.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n for prio in self.__priorities:\n comp = self.__components[prio]\n comp.add_statistics(stat_col)\n\n # Check number of losses in the pipeline.\n num_losses = 0\n for loss in self.losses:\n num_losses += len(loss.loss_keys())\n self.show_total_loss = (num_losses > 1)\n\n # Additional \"total loss\" (for single- and multi-loss pipelines).\n # Collect it always, but show it only for multi-loss pipelines.\n if self.show_total_loss:\n stat_col.add_statistics(\"total_loss\", '{:12.10f}')\n else:\n stat_col.add_statistics(\"total_loss\", None)\n stat_col.add_statistics(\"total_loss_support\", None)\n\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics for every component in the pipeline.\n\n :param stat_col: :py:class:`ptp.utils.StatisticsCollector`.\n\n :param data_streams: ``DataStreams`` containing inputs, targets etc.\n :type data_streams: :py:class:`ptp.data_types.DataStreams`\n\n \"\"\"\n for prio in self.__priorities:\n comp = self.__components[prio]\n comp.collect_statistics(stat_col, data_streams)\n\n # Additional \"total loss\" (for single- and multi-loss pipelines).\n loss_sum = 0\n for loss in self.losses:\n for key in loss.loss_keys():\n loss_sum += data_streams[key].cpu().item()\n stat_col[\"total_loss\"] = loss_sum\n stat_col[\"total_loss_support\"] = len(data_streams[\"indices\"]) # batch size\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Aggregates statistics by calling adequate aggregation method of every component in the pipeline.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n for prio in self.__priorities:\n comp = self.__components[prio]\n comp.add_aggregators(stat_agg)\n\n # Additional \"total loss\" (for single- and multi-loss pipelines).\n # Collect it always, but show it only for multi-loss pipelines.\n if self.show_total_loss:\n stat_agg.add_aggregator(\"total_loss\", '{:12.10f}') \n else:\n stat_agg.add_aggregator(\"total_loss\", None) \n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates statistics by calling adequate aggregation method of every component in the pipeline.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n for prio in self.__priorities:\n comp = self.__components[prio]\n comp.aggregate_statistics(stat_col, stat_agg)\n\n # Additional \"total loss\" (for single- and multi-loss pipelines).\n total_losses = stat_col[\"total_loss\"]\n supports = stat_col[\"total_loss_support\"]\n\n # Special case - no samples!\n if sum(supports) == 0:\n stat_agg.aggregators[\"total_loss\"] = 0\n else: \n # Calculate default aggregate - weighted mean.\n stat_agg.aggregators[\"total_loss\"] = average(total_losses, weights=supports)\n" }, { "path": "ptp/application/sampler_factory.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport numpy as np\n\nimport torch.utils.data.sampler as pt_samplers\nimport ptp.utils.samplers as ptp_samplers \n\nimport ptp.utils.logger as logging\nfrom ptp.configuration.configuration_error import ConfigurationError\n\n\nclass SamplerFactory(object):\n \"\"\"\n Class returning sampler depending on the name provided in the \\\n list of parameters.\n \"\"\"\n\n @staticmethod\n def build(task, config, task_subset_name):\n \"\"\"\n Static method returning particular sampler, depending on the name \\\n provided in the list of parameters & the specified task class.\n\n :param task: Instance of an object derived from the Task class.\n :type task: ``tasks.Task``\n\n :param config: Parameters used to instantiate the sampler.\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n :param task_subset_name: Name of task subset (and associated TaskManager object)\n\n ..note::\n\n ``config`` should contains the exact (case-sensitive) class name of the sampler to instantiate.\n\n\n .. warning::\n\n ``torch.utils.data.sampler.BatchSampler``, \\\n ``torch.utils.data.sampler.DistributedSampler`` are not supported yet.\n\n .. note::\n\n ``torch.utils.data.sampler.SubsetRandomSampler`` expects 'indices' to index a subset of the dataset. \\\n Currently, the user can specify these indices using one of the following options:\n\n - Option 1: range.\n >>> indices = range(20)\n\n - Option 2: range as str.\n >>> range_str = '0, 20'\n\n - Option 3: list of indices.\n >>> yaml_list = yaml.load('[0, 2, 5, 10]')\n\n - Option 4: name of the file containing indices.\n >>> filename = \"~/data/mnist/training_indices.txt\"\n\n .. note::\n\n ``torch.utils.data.sampler.WeightedRandomSampler`` expercse additional parameter 'weights'.\n\n :return: Instance of a given sampler or ``None`` if the section not present or couldn't build the sampler.\n\n \"\"\"\n # Initialize logger.\n logger = logging.initialize_logger('SamplerFactory')\n\n try: \n # Check presence of the typename attribute.\n if 'type' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'type'\")\n\n # Get the class typename.\n typename = config['type']\n logger.info('Trying to instantiate the {} sampler object'.format(typename))\n\n ###########################################################################\n # Handle first special case: SubsetRandomSampler.\n if typename == 'SubsetRandomSampler':\n\n # Check presence of the typename attribute.\n if 'indices' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'indices' \"\n \"required by SubsetRandomSampler\")\n\n # Get and process the indices.\n indices = config['indices']\n\n # Analyze the type.\n if type(indices) == str:\n # Try to open the file.\n try:\n # from expanduser()'s doc: If the expansion fails or if the path does not begin\n # with a tilde, the path is returned unchanged. -> So operation below should be safe.\n file = open(os.path.expanduser(indices), \"r\")\n # Read the file.\n indices = file.readline() \n file.close()\n\n except Exception:\n # Ok, this is not a file.\n pass\n finally:\n # Try to process it as a string.\n # Get the digits.\n digits = indices.split(',')\n indices = [int(x) for x in digits]\n else:\n # Assume that type(indices) is a list of ints.\n digits = indices\n\n # Finally, we got the list of digits.\n if len(digits) == 2:\n # Create a range.\n indices = range(int(digits[0]), int(digits[1]))\n # Else: use them as they are, including single index.\n\n # Check if indices are within range.\n if max(indices) >= len(task):\n raise ConfigurationError(\"SubsetRandomSampler cannot work properly when indices are out of range ({}) \"\n \"considering that there are {} samples in the task\".format(\n max(indices), len(task)))\n\n # Create the sampler object.\n sampler = pt_samplers.SubsetRandomSampler(indices)\n\n ###########################################################################\n # Handle second special case: WeightedRandomSampler.\n elif typename == 'WeightedRandomSampler':\n\n # Check presence of the attribute.\n if 'weights' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'weights' \"\n \"required by WeightedRandomSampler\")\n\n # Load weights from file.\n weights = np.fromfile(os.path.expanduser(config['weights']), dtype=float, count=-1, sep=',')\n\n # Create sampler class.\n sampler = pt_samplers.WeightedRandomSampler(weights, len(task), replacement=True)\n\n ###########################################################################\n # Handle third special case: kFoldRandomSampler.\n elif typename == 'kFoldRandomSampler':\n\n # Check presence of the attribute.\n if 'folds' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'folds' \"\n \"required by kFoldRandomSampler\")\n\n # Create indices, depending on the fold.\n folds = config[\"folds\"]\n if folds < 2:\n raise ConfigurationError(\"kFoldRandomSampler requires at least two 'folds'\")\n # Get epochs per fold (default: 1).\n epochs_per_fold = config.get(\"epochs_per_fold\", 1)\n\n # Create the sampler object.\n sampler = ptp_samplers.kFoldRandomSampler(len(task), folds, epochs_per_fold, task_subset_name == 'training')\n\n ###########################################################################\n # Handle fourd special case: kFoldWeightedRandomSampler.\n elif typename == 'kFoldWeightedRandomSampler':\n\n # Check presence of the attribute.\n if 'weights' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'weights' \"\n \"required by kFoldWeightedRandomSampler\")\n\n # Load weights from file.\n weights = np.fromfile(os.path.expanduser(config['weights']), dtype=float, count=-1, sep=',')\n\n # Check presence of the attribute.\n if 'folds' not in config:\n raise ConfigurationError(\"The sampler configuration section does not contain the key 'folds' \"\n \"required by kFoldWeightedRandomSampler\")\n\n # Create indices, depending on the fold.\n folds = config[\"folds\"]\n if folds < 2:\n raise ConfigurationError(\"kFoldRandomSampler requires at least two 'folds'\")\n # Get epochs per fold (default: 1).\n epochs_per_fold = config.get(\"epochs_per_fold\", 1)\n\n # Create the sampler object.\n sampler = ptp_samplers.kFoldWeightedRandomSampler(weights, len(task), folds, epochs_per_fold, task_subset_name == 'training')\n\n elif typename in ['BatchSampler', 'DistributedSampler']:\n # Sorry, don't support those. Yet;)\n raise ConfigurationError(\"Sampler Factory currently does not support the '{}' sampler. Please pick one of the others \"\n \"or use defaults random sampling\".format(typename))\n else:\n # Verify that the specified class is in the samplers package.\n if typename not in dir(pt_samplers):\n raise ConfigurationError(\"Could not find the specified class '{}' in the samplers package\".format(typename))\n\n # Get the sampler class.\n sampler_class = getattr(pt_samplers, typename)\n # Create \"regular\" sampler.\n sampler = sampler_class(task)\n\n # Return sampler.\n return sampler\n\n except ConfigurationError as e:\n logger.error(e)\n # Do not continue with invalid sampler.\n exit(-1)\n" }, { "path": "ptp/application/task_manager.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport signal\nimport logging\nimport numpy as np\n\nfrom torch.utils.data import DataLoader\n\nimport ptp\n\nimport ptp.utils.logger as logging\nfrom ptp.utils.app_state import AppState\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.application.component_factory import ComponentFactory\nfrom ptp.application.sampler_factory import SamplerFactory\n\n\nclass TaskManager(object):\n \"\"\"\n Class that instantiates and manages task and associated entities (dataloader, sampler etc.).\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the manager.\n\n :param name: Name of the manager (associated with a given config section e.g. 'training', 'validation').\n\n :param config: 'ConfigInterface' object, referring to one of main sections (training/validation/test/...).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n self.name = name\n self.config = config\n\n # Get access to AppState: for command line args, globals etc.\n self.app_state = AppState()\n\n # Initialize logger.\n self.logger = logging.initialize_logger(self.name) \n\n # Single batch that will be used for validation (for validation task manager).\n self.batch = None\n\n\n def worker_init_fn(self, worker_id):\n \"\"\"\n Function to be called by :py:class:`torch.utils.data.DataLoader` on each worker subprocess, \\\n after seeding and before data loading.\n\n .. note::\n\n Set the ``NumPy`` random seed of the worker equal to the previous NumPy seed + its ``worker_id``\\\n to avoid having all workers returning the same random numbers.\n\n\n :param worker_id: the worker id (in [0, :py:class:`torch.utils.data.DataLoader`.num_workers - 1])\n :type worker_id: int\n\n \"\"\"\n # Set random seed of a worker.\n np.random.seed(seed=np.random.get_state()[1][0] + worker_id)\n\n # Ignores SIGINT signal - what enables \"nice\" termination of dataloader worker threads.\n # https://discuss.pytorch.org/t/dataloader-multiple-workers-and-keyboardinterrupt/9740/2\n signal.signal(signal.SIGINT, signal.SIG_IGN)\n\n\n def build(self, log=True):\n \"\"\"\n Method creates a task on the basis of configuration section.\n\n :param log: Logs information and the detected errors (DEFAULT: TRUE)\n\n :return: number of detected errors\n \"\"\"\n try: \n # Create component.\n component, class_obj = ComponentFactory.build(\"task\", self.config[\"task\"])\n\n # Check if class is derived (even indirectly) from Task.\n if not ComponentFactory.check_inheritance(class_obj, ptp.Task.__name__):\n raise ConfigurationError(\"Class '{}' is not derived from the Task class!\".format(class_obj.__name__)) \n\n # Set task.\n self.task = component\n\n # Try to build the sampler.\n # Check if sampler is required, i.e. 'sampler' section is empty.\n if \"sampler\" not in self.config:\n self.logger.info(\"The sampler configuration section is not present, using default 'random' sampling\")\n # Set sampler to none.\n self.sampler = None\n else:\n self.sampler = SamplerFactory.build(self.task, self.config[\"sampler\"], self.name)\n # Set shuffle to False - REQUIRED as those two are exclusive.\n self.config['dataloader'].add_config_params({'shuffle': False})\n\n # build the DataLoader on top of the validation task\n self.dataloader = DataLoader(dataset=self.task,\n batch_size=self.config['task']['batch_size'],\n shuffle=self.config['dataloader']['shuffle'],\n sampler=self.sampler,\n batch_sampler= None,\n num_workers=self.config['dataloader']['num_workers'],\n collate_fn=self.task.collate_fn,\n pin_memory=self.config['dataloader']['pin_memory'],\n drop_last=self.config['dataloader']['drop_last'],\n timeout=self.config['dataloader']['timeout'],\n worker_init_fn=self.worker_init_fn)\n\n # Display sizes.\n if log:\n self.logger.info(\"Task for '{}' loaded (size: {})\".format(self.name, len(self.task)))\n if (self.sampler is not None):\n self.logger.info(\"Sampler for '{}' created (size: {})\".format(self.name, len(self.sampler)))\n\n # Ok, success.\n return 0\n\n except ConfigurationError as e:\n if log:\n self.logger.error(\"Detected configuration error while creating the task instance:\\n {}\".format(e))\n # Return error.\n return 1\n except KeyError as e:\n if log:\n self.logger.error(\"Detected key error while creating the task instance: required key {} is missing\".format(e))\n # Return error.\n return 1\n\n\n def __len__(self):\n \"\"\"\n Returns total number of samples, calculated depending on the settings (batch size, dataloader, drop last etc.).\n \"\"\"\n if self.dataloader.drop_last:\n # if we are supposed to drop the last (incomplete) batch.\n total_num_samples = len(self.dataloader) * self.dataloader.batch_size\n elif self.sampler is not None:\n total_num_samples = len(self.sampler)\n else:\n total_num_samples = len(self.task)\n\n return total_num_samples\n\n\n def get_epoch_size(self):\n \"\"\"\n Compute the number of iterations ('episodes') to run given the size of the dataset and the batch size to cover\n the entire dataset once.\n\n Takes into account whether one used sampler or not.\n\n .. note::\n\n If the last batch is incomplete we are counting it in when ``drop_last`` in ``DataLoader()`` is set to Ttrue.\n\n .. warning::\n\n Leaving this method 'just in case', in most cases one might simply use ''len(dataloader)''.\n\n :return: Number of iterations to perform to go though the entire dataset once.\n\n \"\"\"\n # \"Estimate\" dataset size.\n if (self.sampler is not None):\n task_size = len(self.sampler)\n else:\n task_size = len(self.task)\n\n # If task_size is a multiciplity of batch_size OR drop last is set.\n if (task_size % self.dataloader.batch_size) == 0 or self.dataloader.drop_last:\n return task_size // self.dataloader.batch_size\n else:\n return (task_size // self.dataloader.batch_size) + 1\n\n def initialize_epoch(self):\n \"\"\"\n Function called to initialize a new epoch.\n \"\"\"\n epoch = self.app_state.epoch\n # Update task settings depending on the epoch.\n self.task.initialize_epoch(epoch)\n\n # Generate a single batch used for partial validation.\n if self.name == 'validation':\n if self.batch is None or (self.sampler is not None and \"kFold\" in type(self.sampler).__name__):\n self.batch = next(iter(self.dataloader))\n # TODO refine partial validation section.\n # partial_validation:\n # interval: 100 # How often to test.\n # resample_at_epoch: True # at the beginning of new epoch.\n\n def finalize_epoch(self):\n \"\"\"\n Function called at the end of an epoch to finalize it.\n \"\"\"\n epoch = self.app_state.epoch\n self.task.initialize_epoch(epoch)\n" }, { "path": "ptp/components/component.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport abc\n\nimport ptp.utils.logger as logging\n\nfrom ptp.utils.app_state import AppState\nfrom ptp.utils.globals_facade import GlobalsFacade\nfrom ptp.utils.key_mappings_facade import KeyMappingsFacade\n\nfrom ptp.configuration.config_parsing import load_class_default_config_file\n\n\nclass Component(abc.ABC):\n def __init__(self, name, class_type, config):\n \"\"\"\n Initializes the component. This constructor:\n\n - sets the access to ``AppState`` (for dtypes, settings, globals etc.)\n - stores the component name and type\n - stores reference to the passed configuration registry section\n - loads default component parameters\n - initializes the logger\n - initializes mapping facilities and facades\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n self.name = name\n self.config = config\n\n # Get access to AppState: for command line args, globals etc.\n self.app_state = AppState()\n\n # Initialize logger.\n self.logger = logging.initialize_logger(self.name) \n\n # Load default configuration.\n if class_type is not None:\n self.config.add_default_params(load_class_default_config_file(class_type))\n\n # Initialize the \"streams mapping facility\".\n if \"streams\" not in config or config[\"streams\"] is None:\n self.__stream_keys = {}\n else:\n self.__stream_keys = config[\"streams\"]\n self.stream_keys = KeyMappingsFacade(self.__stream_keys)\n\n # Initialize the \"globals mapping facility\".\n if \"globals\" not in config or config[\"globals\"] is None:\n self.__global_keys = {}\n else:\n self.__global_keys = config[\"globals\"]\n self.global_keys = KeyMappingsFacade(self.__global_keys)\n\n # Initialize the \"statistics mapping facility\".\n if \"statistics\" not in config or config[\"statistics\"] is None:\n self.__statistics_keys = {}\n else:\n self.__statistics_keys = config[\"statistics\"]\n self.statistics_keys = KeyMappingsFacade(self.__statistics_keys)\n\n # Facade for accessing global parameters (stored still in AppState).\n self.globals = GlobalsFacade(self.__global_keys)\n\n\n def summarize_io(self, priority = -1):\n \"\"\"\n Summarizes the component by showing its name, type and input/output definitions.\n\n :param priority: Component priority (DEFAULT: -1)\n\n :return: Summary as a str.\n\n \"\"\"\n summary_str = \" + {} ({}) [{}] \\n\".format(self.name, type(self).__name__, priority)\n # Get inputs\n summary_str += ' Inputs:\\n' \n for key,value in self.input_data_definitions().items():\n summary_str += ' {}: {}, {}, {}\\n'.format(key, value.dimensions, value.types, value. description)\n # Get outputs.\n summary_str += ' Outputs:\\n' \n for key,value in self.output_data_definitions().items():\n summary_str += ' {}: {}, {}, {}\\n'.format(key, value.dimensions, value.types, value. description)\n # Return string.\n return summary_str\n\n\n @abc.abstractmethod\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n Abstract, must be implemented by all derived classes.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.configuration.DataDefinition`).\n \"\"\"\n pass\n\n @abc.abstractmethod\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n Abstract, must be implemented by all derived classes.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.configuration.DataDefinition`).\n \"\"\"\n pass\n\n def handshake_input_definitions(self, all_definitions, log_errors=True):\n \"\"\" \n Checks whether all_definitions contain fields required by the given component.\n\n :param all_definitions: dictionary containing output data definitions (each of type :py:class:`ptp.configuration.DataDefinition`).\n\n :param log_errors: Logs the detected errors (DEFAULT: TRUE)\n\n :return: number of detected errors.\n \"\"\"\n errors = 0\n for (key,id) in self.input_data_definitions().items():\n # Check presence of key.\n if key not in all_definitions.keys():\n if log_errors:\n self.logger.error(\"Input definition: expected field '{}' not found in DataStreams keys ({})\".format(key, all_definitions.keys()))\n errors += 1\n continue\n # Check number of dimensions.\n dd = all_definitions[key]\n if len(id.dimensions) != len (dd.dimensions):\n if log_errors:\n self.logger.error(\"Input definition: field '{}' in DataStreams has different dimensions from expected (expected {} while received {})\".format(key, id.dimensions, dd.dimensions))\n errors += 1\n else: \n # Check dimensions one by one.\n for index, (did, ddd) in enumerate(zip(id.dimensions, dd.dimensions)):\n # -1 means that it can handle different values here.\n if did != -1 and did != ddd:\n if log_errors:\n self.logger.error(\"Input definition: field '{}' in DataStreams has dimension {} different from expected (expected {} while received {})\".format(key,index, id.dimensions, dd.dimensions))\n errors += 1\n # Check number of types.\n if len(id.types) != len (dd.types):\n if log_errors:\n self.logger.error(\"Input definition: field '{}' in DataStreams has number of types different from expected (expected {} while received {})\".format(key, id.types, dd.types))\n errors += 1\n else: \n # Check types one by one.\n for index, (tid, tdd) in enumerate(zip(id.types, dd.types)):\n # -1 means that it can handle different values here.\n if tid != tdd:\n if log_errors:\n self.logger.error(\"Input definition: field '{}' in DataStreams has type {} different from expected (expected {} while received {})\".format(key,index, id.types, dd.types))\n errors += 1\n\n return errors\n \n def export_output_definitions(self, all_definitions, log_errors=True):\n \"\"\" \n Exports output definitions to all_definitions, checking errors (e.g. if output field is already present in all_definitions).\n\n :param all_definitions: dictionary containing output data definitions (each of type :py:class:`ptp.configuration.DataDefinition`).\n\n :param log_errors: Logs the detected errors (DEFAULT: TRUE)\n\n :return: number of detected errors.\n \"\"\"\n errors = 0\n for (key,od) in self.output_data_definitions().items():\n # Check presence of key.\n if key in all_definitions.keys():\n if log_errors:\n self.logger.error(\"Output definition error: field '{}' cannot be added to DataStreams, as it is already present in its keys ({})\".format(key, all_definitions.keys()))\n errors += 1\n else:\n # Add field to definitions.\n all_definitions[key] = od\n\n return errors\n\n\n @abc.abstractmethod\n def __call__(self, data_streams):\n \"\"\"\n Method responsible for processing the data dict.\n Abstract, must be implemented by all derived classes.\n\n :param data_streams: :py:class:`ptp.data_types.DataStreams` object containing both input data to be processed and that will be extended by the results.\n \"\"\"\n pass\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds statistics to :py:class:`ptp.configuration.StatisticsCollector`.\n\n .. note::\n\n Empty - To be redefined in inheriting classes.\n\n :param stat_col: :py:class:`ptp.configuration.StatisticsCollector`.\n\n \"\"\"\n pass\n\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Base statistics collection.\n\n .. note::\n\n Empty - To be redefined in inheriting classes. The user has to ensure that the corresponding entry \\\n in the :py:class:`ptp.configuration.StatisticsCollector` has been created with \\\n :py:func:`add_statistics` beforehand.\n\n :param stat_col: :py:class:`ptp.configuration.StatisticsCollector`.\n\n :param data_streams: ``DataStreams`` containing inputs, targets etc.\n :type data_streams: :py:class:`ptp.data_types.DataStreams`\n\n \"\"\"\n pass\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds statistical aggregators to :py:class:`ptp.configuration.StatisticsAggregator`.\n\n .. note::\n\n Empty - To be redefined in inheriting classes.\n\n :param stat_agg: :py:class:`ptp.configuration.StatisticsAggregator`.\n\n \"\"\"\n pass\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates the statistics collected by :py:class:`ptp.configuration.StatisticsCollector` and adds the \\\n results to :py:class:`ptp.configuration.StatisticsAggregator`.\n\n .. note::\n\n Empty - To be redefined in inheriting classes.\n The user can override this function in subclasses but should call \\\n :py:func:`aggregate_statistics` to collect basic statistical aggregators (if set).\n\n :param stat_col: :py:class:`ptp.configuration.StatisticsCollector`.\n\n :param stat_agg: :py:class:`ptp.configuration.StatisticsAggregator`.\n\n \"\"\"\n pass\n" }, { "path": "ptp/components/language/__init__.py", "content": "from .bow_encoder import BOWEncoder\nfrom .label_indexer import LabelIndexer\nfrom .sentence_indexer import SentenceIndexer\nfrom .sentence_one_hot_encoder import SentenceOneHotEncoder\nfrom .sentence_tokenizer import SentenceTokenizer\nfrom .word_decoder import WordDecoder\n\n__all__ = [\n 'BOWEncoder',\n 'LabelIndexer',\n 'SentenceIndexer',\n 'SentenceOneHotEncoder',\n 'SentenceTokenizer',\n 'WordDecoder'\n ]\n" }, { "path": "ptp/components/language/bow_encoder.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass BOWEncoder(Component):\n \"\"\"\n Simple Bag-of-word type encoder that encodes the sentence (in the form of a list of encoded words) into a vector.\n \n .. warning::\n BoW transformation is inreversible, thus decode-related methods in fact return original inputs.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the bag-of-word encoded by creating dictionary mapping ALL words from training, validation and test sets into unique indices.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, BOWEncoder, config)\n\n # Default name mappings for all encoders.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve bow size from global variables.\n self.bow_size = self.globals[\"bow_size\"]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, -1, self.bow_size], [list, list, torch.Tensor], \"Batch of sentences, each represented as a list of vectors [BATCH_SIZE] x [SEQ_LENGTH] x [ITEM_SIZE] (agnostic to item size)\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.bow_size], [torch.Tensor], \"Batch of sentences, each represented as a single vector [BATCH_SIZE x ITEM_SIZE] (agnostic to item size)\")\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes batch, or, in fact, only one field of batch (\"inputs\").\n Stores result in \"outputs\" field of data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input containing list of (list of tokens) [BATCH SIZE] x [SEQ_LEN] x [ITEM_SIZE]\n - \"outputs\": added output tensor with encoded words [BATCH_SIZE x ITEM_SIZE]\n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n outputs_list = []\n # Process samples 1 by one.\n for sample in inputs:\n # Encode sample\n output = self.encode_sample(sample)\n # Add to list plus unsqueeze inputs dimension(!)\n outputs_list.append( output.unsqueeze(0) )\n # Concatenate output tensors.\n outputs = torch.cat(outputs_list, dim=0)\n # Add result to the data dict.\n data_streams.publish({self.key_outputs: outputs})\n\n def encode_sample(self, list_of_tokens):\n \"\"\"\n Generates a bag-of-word vector of length `bow_size`.\n\n :param list_of_tokens: List of tokens [SEQ_LENGTH] x [ITEM_SIZE]\n :return: torch.LongTensor [ITEM_SIZE]\n \"\"\"\n # Create output.\n output = list_of_tokens[0]\n # \"Adds\" tokens.\n for token in list_of_tokens[1:]:\n output += token\n return output\n" }, { "path": "ptp/components/language/label_indexer.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.components.mixins.word_mappings import WordMappings\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass LabelIndexer(Component, WordMappings):\n \"\"\"\n Class responsible for changing of samples consisting of single words/labels into indices (that e.g. can be latter used for loss calculation, PyTorch-style).\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the component.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructor(s) of parent class(es) - in the right order!\n Component.__init__(self, name, LabelIndexer, config)\n WordMappings.__init__(self)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Get value from configuration.\n self.out_of_vocabulary_value = self.config[\"out_of_vocabulary_value\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, 1], [list, str], \"Batch of labels (words), each represented as a single string [BATCH_SIZE] x [string]\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1], [torch.Tensor], \"Batch of labels, each represented as a single index [BATCH_SIZE]\")\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single word.\n Stores result in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing list of words [BATCH_SIZE] x x [string]\n\n - \"outputs\": added output field containing list of indices [BATCH_SIZE] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n outputs_list = []\n # Process samples 1 by 1.\n for sample in inputs:\n assert not isinstance(sample, (list,)), 'This encoder requires input sample to contain a single word'\n # Process single token.\n if sample in self.word_to_ix.keys():\n output_sample = self.word_to_ix[sample]\n else:\n # Word out of vocabulary.\n output_sample = self.out_of_vocabulary_value\n outputs_list.append(output_sample)\n # Transform to tensor.\n output_tensor = torch.tensor(outputs_list)\n # Create the returned dict.\n data_streams.publish({self.key_outputs: output_tensor})\n" }, { "path": "ptp/components/language/sentence_indexer.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.components.mixins.word_mappings import WordMappings, pad_trunc_list\n\n\nclass SentenceIndexer(Component, WordMappings):\n \"\"\"\n Class responsible for encoding of sequences of words into list of indices.\n Those can be letter embedded, encoded with 1-hot encoding or else.\n\n Additianally, when 'reverse' mode is on, it works in the oposite direction, i.e. changing tensor witl indices into list of words.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the component.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructor(s) of parent class(es) - in the right order!\n Component.__init__(self, name, SentenceIndexer, config)\n WordMappings.__init__(self)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Read mode from the configuration.\n self.mode_reverse = self.config['reverse']\n\n # Force padding to a fixed length\n self.fixed_padding = self.config['fixed_padding']\n\n # Wether to add at the end of sequence\n self.enable_eos_token = self.config['eos_token']\n\n if self.mode_reverse:\n # We will need reverse (index:word) mapping.\n self.ix_to_word = dict((v,k) for k,v in self.word_to_ix.items())\n\n # Get inputs distributions/indices flag.\n self.use_input_distributions = self.config[\"use_input_distributions\"]\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n if self.mode_reverse:\n if self.use_input_distributions:\n return {\n self.key_inputs: DataDefinition([-1, -1, -1], [torch.Tensor], \"Batch of sentences represented as a single tensor with batch of probability distributions [BATCH_SIZE x SEQ_LENGTH x ITEM_SIZE]\"),\n }\n else: \n return {\n self.key_inputs: DataDefinition([-1, -1], [torch.Tensor], \"Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]\"),\n }\n else: \n return {\n self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n if self.mode_reverse:\n return {\n self.key_outputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\"),\n }\n else: \n return {\n self.key_outputs: DataDefinition([-1, -1], [torch.Tensor], \"Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]\"),\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes inputs into outputs.\n Depending on the mode (set by 'reverse' config param) calls sentences_to_tensor() (when False) or tensor_to_sentences() (when set to True).\n\n :param data_streams: :py:class:`ptp.datatypes.DataStreams` object.\n \"\"\"\n if self.mode_reverse:\n if self.use_input_distributions:\n # Produce list of words.\n self.tensor_distributions_to_sentences(data_streams)\n else:\n # Produce list of words.\n self.tensor_indices_to_sentences(data_streams)\n else:\n # Produce indices.\n self.sentences_to_tensor(data_streams)\n\n\n def sentences_to_tensor(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of batch of list of words into a single tensor with corresponding indices.\n\n :param data_streams: :py:class:`ptp.datatypes.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing list of lists of words [BATCH_SIZE] x [SEQ_SIZE] x [string]\n\n - \"outputs\": added output field containing tensor with indices [BATCH_SIZE x SEQ_SIZE] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n\n # Get index of padding.\n pad_index = self.word_to_ix['']\n eos_index = self.word_to_ix[''] if self.enable_eos_token else None\n\n outputs_list = []\n # Process sentences 1 by 1.\n for sample in inputs:\n assert isinstance(sample, (list,)), 'This encoder requires input sample to contain a list of words'\n # Process list.\n output_sample = []\n # Encode sample (list of words)\n for token in sample:\n # Get index.\n output_index = self.word_to_ix[token]\n # Add index to outputs.\n output_sample.append( output_index )\n\n # Apply fixed padding to all sequences if requested\n # Otherwise let torch.nn.utils.rnn.pad_sequence handle it and choose a dynamic padding\n if self.fixed_padding > 0:\n pad_trunc_list(output_sample, self.fixed_padding, padding_value=pad_index, eos_value=eos_index)\n\n outputs_list.append(self.app_state.LongTensor(output_sample))\n\n # Transform the list of lists to tensor.\n # output = self.app_state.LongTensor(outputs_list)\n output = torch.nn.utils.rnn.pad_sequence(outputs_list, batch_first=True, padding_value=pad_index)\n # Create the returned dict.\n data_streams.publish({self.key_outputs: output})\n\n def tensor_indices_to_sentences(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of tensor with indices into a batch of list of words.\n\n :param data_streams: :py:class:`ptp.datatypes.DataStreams` object containing (among others):\n\n - \"inputs\": added output field containing tensor with indices [BATCH_SIZE x SEQ_SIZE] \n\n - \"outputs\": expected input field containing list of lists of words [BATCH_SIZE] x [SEQ_SIZE] x [string]\n\n \"\"\"\n # Get inputs to be changed to words.\n inputs = data_streams[self.key_inputs].data.cpu().numpy().tolist()\n\n outputs_list = []\n # Process samples 1 by 1.\n for sample in inputs:\n # Process list.\n output_sample = []\n # \"Decode\" sample (list of indices).\n for token in sample:\n # Get word.\n output_word = self.ix_to_word[token]\n # Add index to outputs.\n output_sample.append( output_word )\n\n # Add sentence to batch.\n outputs_list.append(output_sample)\n\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs_list})\n\n def tensor_distributions_to_sentences(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of tensor with probability distributions into a batch of list of words.\n\n :param data_streams: :py:class:`ptp.datatypes.DataStreams` object containing (among others):\n\n - \"inputs\": added output field containing tensor with indices [BATCH_SIZE x SEQ_SIZE x ITEM_SIZE] \n\n - \"outputs\": expected input field containing list of lists of words [BATCH_SIZE] x [SEQ_SIZE] x [string]\n\n \"\"\"\n # Get inputs to be changed to words.\n inputs = data_streams[self.key_inputs].max(2)[1].data.cpu().numpy().tolist()\n\n outputs_list = []\n # Process samples 1 by 1.\n for sample in inputs:\n # Process list.\n output_sample = []\n # \"Decode\" sample (list of indices).\n for token in sample:\n\n # Get word.\n output_word = self.ix_to_word[token]\n # Add index to outputs.\n output_sample.append( output_word )\n\n # Add sentence to batch.\n outputs_list.append(output_sample)\n\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs_list})\n" }, { "path": "ptp/components/language/sentence_one_hot_encoder.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.components.mixins.word_mappings import WordMappings\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass SentenceOneHotEncoder(Component, WordMappings):\n \"\"\"\n Class responsible for encoding of samples being sequences of words using 1-hot encoding.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the component.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructor(s) of parent class(es) - in the right order!\n Component.__init__(self, name, SentenceOneHotEncoder, config)\n WordMappings.__init__(self)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, -1, len(self.word_to_ix)], [list, list, torch.Tensor], \"Batch of sentences, each represented as a list of vectors [BATCH_SIZE] x [SEQ_LENGTH] x [VOCABULARY_SIZE]\"),\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of list of tokens (for a single sample)\n Stores result in \"encoded_inputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing list of words [BATCH_SIZE] x [SEQ_SIZE] x [string]\n\n - \"encoded_targets\": added output field containing list of indices [BATCH_SIZE] x [SEQ_SIZE] x [VOCABULARY_SIZE1] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n outputs_list = []\n # Process samples 1 by one.\n for sample in inputs:\n assert isinstance(sample, (list,)), 'This encoder requires input sample to contain a list of words'\n # Process list.\n output_sample = []\n # Encode sample (list of words)\n for token in sample:\n # Create empty vector.\n output_token = torch.zeros(len(self.word_to_ix)).type(self.app_state.FloatTensor)\n # Add one for given word\n output_token[self.word_to_ix[token]] += 1\n # Add to outputs.\n output_sample.append( output_token )\n\n outputs_list.append(output_sample)\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs_list})\n" }, { "path": "ptp/components/language/sentence_tokenizer.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport nltk\n#from nltk.tokenize import WhitespaceTokenizer\nimport string\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\nfrom ptp.configuration.config_parsing import get_value_list_from_dictionary\n\n\nclass SentenceTokenizer(Component):\n \"\"\"\n Class responsible for tokenizing the sentence.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the component.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, SentenceTokenizer, config)\n\n # Read the actual configuration.\n self.mode_detokenize = config['detokenize']\n\n # Get preprocessing.\n self.preprocessing = get_value_list_from_dictionary(\n \"preprocessing\", self.config,\n 'none | lowercase | remove_punctuation | all'.split(\" | \")\n )\n if 'none' in self.preprocessing:\n self.preprocessing = []\n if 'all' in self.preprocessing:\n self.preprocessing = 'lowercase | remove_punctuation'.split(\" | \")\n self.logger.info(\"Applied preprocessing: {}\".format(self.preprocessing))\n\n self.remove_characters = get_value_list_from_dictionary(\"remove_characters\", self.config)\n self.logger.info(\"Additional characters that will be removed during preprocessing: {}\".format(self.remove_characters))\n\n if 'remove_punctuation' in self.preprocessing:\n self.translator = str.maketrans('', '', string.punctuation)\n\n # Tokenizer.\n self.tokenizer = nltk.tokenize.WhitespaceTokenizer()\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n if self.mode_detokenize:\n # list of strings -> sentence.\n self.processor = self.detokenize_sample\n else:\n # sentence -> list of strings.\n self.processor = self.tokenize_sample\n # Ok, we are ready to go!\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n if self.mode_detokenize == False:\n return { self.key_inputs: DataDefinition([-1, 1], [list, str], \"Batch of sentences, each represented as a single string [BATCH_SIZE] x [string]\") }\n else:\n return { self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of tokenized sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\") }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n if self.mode_detokenize == False:\n return { self.key_outputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of tokenized sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\") }\n else:\n return { self.key_outputs: DataDefinition([-1, 1], [list, str], \"Batch of sentences, each represented as a single string [BATCH_SIZE] x [string]\") }\n\n\n def tokenize_sample(self, text):\n \"\"\"\n Changes text (sentence) into list of tokens (words).\n\n :param text: sentence (string).\n\n :return: list of words (strings).\n \"\"\"\n # Lowercase.\n if 'lowercase' in self.preprocessing:\n text = text.lower()\n\n # Remove characters.\n for char in self.remove_characters:\n text = text.replace(char, ' ')\n\n # Remove punctuation.\n if 'remove_punctuation' in self.preprocessing:\n text = text.translate(self.translator)\n\n # Tokenize.\n text_words = self.tokenizer.tokenize(text)\n\n return text_words\n\n def detokenize_sample(self, sample):\n \"\"\"\n Changes list of tokens (words) into sentence.\n\n :param sample: list of words (strings).\n\n :return: sentence (string).\n \"\"\"\n return ' '.join([str(x) for x in sample])\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes batch, or, in fact, only one field of bach (\"inputs\").\n Stores result in \"encoded_inputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing list of words\n\n - \"encoded_targets\": added field containing output, tensor with encoded samples [BATCH_SIZE x 1] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n outputs_list = []\n # Process samples 1 by one.\n for sample in inputs:\n output = self.processor(sample)\n # Add to outputs.\n outputs_list.append( output )\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs_list})\n" }, { "path": "ptp/components/language/word_decoder.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.components.mixins.word_mappings import WordMappings\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass WordDecoder(Component, WordMappings):\n \"\"\"\n Class responsible for decoding of samples encoded in the form of vectors (\"probability distributions\").\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the component.\n\n :param name: Component name (read from configuration file).\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructor(s) of parent class(es) - in the right order!\n Component.__init__(self, name, WordDecoder, config)\n WordMappings.__init__(self)\n\n # Construct reverse mapping for faster processing.\n self.ix_to_word = dict((v,k) for k,v in self.word_to_ix.items())\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, -1], [torch.Tensor], \"Batch of words, each represented as a vector (probability distribution) [BATCH_SIZE x ITEM_SIZE] (agnostic to item size)\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, 1], [list, str], \"Batch of words, each represented as a single string [BATCH_SIZE] x [string]\")\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single word.\n Stores result in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ITEM_SIZE]\n\n - \"outputs\": added output field containing list of words [BATCH_SIZE] x [string] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n outputs_list = []\n # Process samples 1 by 1.\n for sample in inputs.chunk(inputs.size(0), 0):\n # Process single token.\n max_index = sample.squeeze(0).argmax(dim=0).item() \n output_sample = self.ix_to_word[max_index]\n outputs_list.append(output_sample)\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs_list})\n\n" }, { "path": "ptp/components/losses/__init__.py", "content": "from .loss import Loss\nfrom .nll_loss import NLLLoss\n\n__all__ = [\n 'NLLLoss',\n 'Loss',\n ]\n" }, { "path": "ptp/components/losses/loss.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\n\n\nclass Loss(Component):\n \"\"\"\n Class representing base class for all losses.\n\n Implements features & attributes used by all subclasses.\n\n \"\"\"\n\n def __init__(self, name, class_type, config):\n \"\"\"\n Initializes loss object.\n\n :param name: Loss name.\n :type name: str\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.utils.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, class_type, config)\n\n # Get key mappings.\n self.key_targets = self.stream_keys[\"targets\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n self.key_loss = self.stream_keys[\"loss\"]\n\n def loss_keys(self):\n \"\"\" \n Function returns a list containing keys used to return losses in DataStreams.\n Those keys will be used to find objects that will be roots for backpropagation of gradients.\n\n :return: list of keys associated with losses returned by the component.\n \"\"\"\n return [ self.key_loss ]\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds most elementary shared statistics to ``StatisticsCollector``: episode and loss.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n # Add loss statistics with formatting.\n stat_col.add_statistics(self.key_loss, '{:12.10f}')\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics (loss) for given episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col[self.key_loss] = data_streams[self.key_loss].item()\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds basic loss-related statistical aggregators to ``StatisticsAggregator``.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n # Add default statistical aggregators for the loss (indicating a formatting).\n # Represents the average loss, but stying with loss for TensorBoard \"variable compatibility\".\n stat_agg.add_aggregator(self.key_loss, '{:12.10f}') \n stat_agg.add_aggregator(self.key_loss+'_min', '{:12.10f}')\n stat_agg.add_aggregator(self.key_loss+'_max', '{:12.10f}')\n stat_agg.add_aggregator(self.key_loss+'_std', '{:12.10f}')\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates the statistics collected by the ``StatisticsCollector``.\n\n .. note::\n Computes min, max, mean, std of the loss as these are basic statistical aggregator by default.\n\n Given that the ``StatisticsAggregator`` uses the statistics collected by the ``StatisticsCollector``, \\\n It should be ensured that these statistics are correctly collected (i.e. use of ``self.add_statistics()`` \\\n and ``collect_statistics()``).\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n # Get loss values.\n loss_values = stat_col[self.key_loss]\n\n # Calculate default aggregates.\n stat_agg.aggregators[self.key_loss] = torch.mean(torch.tensor(loss_values))\n stat_agg.aggregators[self.key_loss+'_min'] = min(loss_values)\n stat_agg.aggregators[self.key_loss+'_max'] = max(loss_values)\n stat_agg.aggregators[self.key_loss+'_std'] = 0.0 if len(loss_values) <= 1 else torch.std(torch.tensor(loss_values))\n" }, { "path": "ptp/components/losses/nll_loss.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nimport torch.nn as nn\n\nfrom ptp.components.losses.loss import Loss\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass NLLLoss(Loss):\n \"\"\"\n Component calculating the negative log likelihood loss.\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object, calls base constructor, initializes names of input and output ports.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n\n \"\"\"\n # Call constructors of parent classes.\n Loss.__init__(self, name, NLLLoss, config)\n\n # Get stream key mappnigs.\n self.key_masks = self.stream_keys[\"masks\"]\n\n # Get masking flag.\n self.use_masking = self.config[\"use_masking\"]\n\n # Get number of targets dimensions.\n self.num_targets_dims = self.config[\"num_targets_dims\"]\n\n # Get the optional ignore_index. -100 is the default value in PyTorch\n self.ignore_index = -100\n try:\n self.ignore_index = self.globals[\"ignore_index\"]\n except KeyError:\n pass\n\n # Set loss.\n self.loss_function = nn.NLLLoss(ignore_index=self.ignore_index)\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n input_defs = {\n self.key_targets: DataDefinition([-1]*self.num_targets_dims, [torch.Tensor], \"Batch of targets (indices) [DIM 1 x DIM 2 x ... ]\"),\n self.key_predictions: DataDefinition([-1]*(self.num_targets_dims+1), [torch.Tensor], \"Batch of predictions, represented as tensor with probability distribution over classes [DIM 1 x DIM x ... x NUM_CLASSES]\")\n }\n if self.use_masking:\n input_defs[self.key_masks] = DataDefinition([-1], [torch.Tensor], \"Batch of masks [BATCH_SIZE]\")\n return input_defs\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_loss: DataDefinition([1], [torch.Tensor], \"Loss value (single value for the whole batch - a scalar)\")\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Calculates loss (negative log-likelihood) and adds it to data dict.\n\n :param data_streams: DataStreams containing:\n - \"targets\": input batch of targets (class indices) [BATCH_SIZE x NUM_CLASSES]\n\n - \"predictions\": input batch of predictions (log_probs) being outputs of the model [BATCH_SIZE x 1]\n\n - \"loss\": output scalar representing the loss.\n\n \"\"\"\n # Load inputs.\n targets = data_streams[self.key_targets]\n predictions = data_streams[self.key_predictions]\n\n #print(\"targets = \",targets)\n #print(\"predictions = \",predictions)\n\n #if isinstance(targets, (list,)):\n # # Change to long tensor, as expected by nllloss.\n # targets = torch.LongTensor(targets)\n\n # Mask predictions if option set.\n\n if self.use_masking:\n masks = data_streams[self.key_masks]\n targets = targets * masks.type(self.app_state.LongTensor)\n #print(\"unsqueezed masks = \", masks.unsqueeze(1))\n predictions = predictions * masks.unsqueeze(1).type(self.app_state.FloatTensor)\n\n #print(\"masked targets = \",targets)\n #print(\"masked predictions = \",predictions)\n \n # reshape.\n last_dim = predictions.size(-1)\n\n #print(\"\\nTarget: {}\\n Prediction: {}\".format(targets.view(-1), predictions.view(-1, last_dim)))\n\n # Calculate loss.\n loss = self.loss_function(predictions.view(-1, last_dim), targets.view(-1))\n # Add it to datadict.\n data_streams.publish({self.key_loss: loss})\n" }, { "path": "ptp/components/masking/__init__.py", "content": "from .join_masked_predictions import JoinMaskedPredictions\nfrom .string_to_mask import StringToMask\n\n__all__ = [\n 'JoinMaskedPredictions',\n 'StringToMask',\n ]\n" }, { "path": "ptp/components/masking/join_masked_predictions.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nimport numpy as np\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass JoinMaskedPredictions(Component):\n \"\"\"\n Class responsible joining several prediction streams using the associated masks.\n Additionally, it returns the associated string indices.\n\n .. warning:\n As performed operations are not differentiable, the returned 'output_indices' cannot be used for e.g. calculation of loss!!\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the object. Loads keys, word mappings and vocabularies.\n\n :param name: Name of the component read from the configuration file\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, JoinMaskedPredictions, config)\n\n # Get input key mappings.\n # Load list of prediction streams names (keys).\n self.input_prediction_stream_keys = self.config[\"input_prediction_streams\"]\n if type(self.input_prediction_stream_keys) == str:\n self.input_prediction_stream_keys = self.input_prediction_stream_keys.replace(\" \", \"\").split(\",\")\n #assert(self.input_prediction_stream_keys != \"\"), \"ooo\"\n\n # Load list of mask streams names (keys).\n self.input_mask_stream_keys = self.config[\"input_mask_streams\"]\n if type(self.input_mask_stream_keys) == str:\n self.input_mask_stream_keys = self.input_mask_stream_keys.replace(\" \", \"\").split(\",\")\n\n # Load list of word mappings names (keys).\n input_word_mappings_keys = self.config[\"input_word_mappings\"]\n if type(input_word_mappings_keys) == str:\n input_word_mappings_keys = input_word_mappings_keys.replace(\" \", \"\").split(\",\")\n\n # Retrieve input word mappings from globals.\n self.input_ix_to_word = []\n for wmk in input_word_mappings_keys:\n # Get word mappings.\n word_to_ix = self.globals[wmk]\n # Create inverse transformation.\n ix_to_word = {value: key for (key, value) in word_to_ix.items()}\n self.input_ix_to_word.append(ix_to_word)\n\n\n # Get output key mappings.\n self.key_output_indices = self.stream_keys[\"output_indices\"]\n self.key_output_strings = self.stream_keys[\"output_strings\"]\n\n # Retrieve output word mappings from globals.\n self.output_word_to_ix = self.globals[\"output_word_mappings\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n input_defs = {}\n # Add input prediction streams.\n for i, ipsk in enumerate(self.input_prediction_stream_keys):\n # Use input prediction stream key along with the length of the associated word mappings (i.e. size of the vocabulary = NUM_CLASSES)\n input_defs[ipsk] = DataDefinition([-1, len(self.input_ix_to_word[i])], [torch.Tensor], \"Batch of predictions, represented as tensor with probability distribution over classes [BATCH_SIZE x NUM_CLASSES]\")\n # Add mask streams.\n for imsk in self.input_mask_stream_keys:\n # Every mask has the same definition, but different stream key.\n input_defs[imsk] = DataDefinition([-1], [torch.Tensor], \"Batch of masks [BATCH_SIZE]\")\n\n return input_defs\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {\n self.key_output_indices: DataDefinition([-1], [torch.Tensor], \"Batch of merged (output) indices [BATCH_SIZE]\"),\n self.key_output_strings: DataDefinition([-1, 1], [list, str], \"Batch of merged strings, corresponging to indices when using the provided word mappings [BATCH_SIZE] x [STRING]\")\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs masks.\n masks = []\n for imsk in self.input_mask_stream_keys:\n # Get mask from stream.\n mask = data_streams[imsk]\n masks.append(mask.data.cpu().numpy())\n \n # Sum all masks and make sure that they are complementary.\n masks_sum = np.sum(masks, axis=0)\n batch_size = masks_sum.shape[0]\n sum_ones = sum(filter(lambda x: x == 1, masks_sum))\n if sum_ones != batch_size:\n self.logger.error(\"Masks received from the {} streams are not complementary!\".format(self.input_mask_stream_keys))\n exit(-1)\n\n # Create mapping indicating from which input prediction/mask/dictionary stream we will take data associated with given \"sample\".\n weights = np.array(range(len(masks)))\n masks = np.array(masks).transpose()\n mapping = np.dot(masks, weights)\n #print(\"Mapping = \\n\",mapping)\n\n # \"Translate\". \n output_answers = []\n output_indices = []\n output_predictions_lst = []\n # Iterate through samples.\n for sample in range(batch_size):\n # Get the right dictionary.\n ix_to_word = self.input_ix_to_word[mapping[sample]]\n #print(ix_to_word)\n\n # Get the right sample from the right prediction stream.\n sample_prediction = data_streams[self.input_prediction_stream_keys[mapping[sample]]][sample]\n #print(sample_prediction)\n output_predictions_lst.append(sample_prediction)\n\n # Get the index of max log-probabilities.\n index = sample_prediction.max(0)[1].data.cpu().item()\n #print(index)\n \n # Get the right word.\n word = ix_to_word[index]\n output_answers.append(word)\n\n # Get original index using output dictionary.\n output_indices.append(self.output_word_to_ix[word])\n\n #print(output_predictions_lst)\n #targets = data_streams[\"targets\"].data.cpu().numpy()\n #print(\"targets = \\n\",targets.tolist())\n #print(\"joined answers = \\n\",output_indices)\n\n # Change to tensor.\n output_indices_tensor = torch.tensor(output_indices)\n\n # Extend the dict by returned output streams.\n data_streams.publish({\n self.key_output_indices: output_indices_tensor,\n self.key_output_strings: output_answers\n })\n \n" }, { "path": "ptp/components/masking/string_to_mask.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass StringToMask(Component):\n \"\"\"\n Class responsible for producing masks for strings using the provided word mappings.\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object. Loads key and word mappings.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, StringToMask, config)\n\n # Get key mappings.\n self.key_strings = self.stream_keys[\"strings\"]\n self.key_masks = self.stream_keys[\"masks\"]\n \n # Retrieve word mappings from globals.\n self.word_to_ix = self.globals[\"word_mappings\"]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n self.key_strings: DataDefinition([-1, 1], [list, str], \"Batch of strings, each being treated as a single 'vocabulary entry' (word) [BATCH_SIZE] x [STRING]\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {\n self.key_masks: DataDefinition([-1], [torch.Tensor], \"Batch of masks [BATCH_SIZE]\")\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs strings.\n strings = data_streams[self.key_strings]\n\n masks = torch.zeros(len(strings), requires_grad=False).type(self.app_state.ByteTensor)\n\n # Process samples 1 by 1.\n for i,sample in enumerate(strings):\n assert not isinstance(sample, (list,)), \"This masking component requires input 'string' to contain a single word\"\n # Process single token.\n if sample in self.word_to_ix.keys():\n masks[i] = 1\n \n # Extend the dict by returned stream with masks.\n data_streams.publish({\n self.key_masks: masks\n })\n\n" }, { "path": "ptp/components/mixins/embeddings.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport numpy as np\nimport tqdm\n\nimport torch\n\nimport ptp.components.mixins.io as io\n\n\ndef load_pretrained_embeddings(logger, folder, embeddings_name, word_to_ix, embeddings_size):\n \"\"\"\n Creates embedding vector for words from the provided (word:index) mappings (dictionary).\n\n Loads the pretrained embeddings from the GloVe project - for the words found in the dictionary.\n \n For words out of dictionary initializes random vectors.\n\n Available embeddings:\n - glove.6B.50d.txt\n - glove.6B.100d.txt\n - glove.6B.200d.txt\n - glove.6B.300d.txt\n - glove.42B.300d.txt\n - glove.840B.300d.txt\n - glove.twitter.27B.txt\n - fasttext.mimic.300d.txt\n\n :param logger: Logger object.\n\n :param folder: Relative path to to the folder.\n :type folder: str\n\n :param word_to_ix: (word:index) mappings\n :type word_to_ix: dict\n\n :param embeddings_size: Embeddings size. Warning: must match the length of vector in the selected file.\n\n :return: Torch tensor with loaded (or random) vectors.\n \"\"\"\n # https://medium.com/@martinpella/how-to-use-pre-trained-word-embeddings-in-pytorch-71ca59249f76\n # http://ronny.rest/blog/post_2017_08_04_glove/\n\n # Check th presence of the file.\n # Available options.\n # https://nlp.stanford.edu/projects/glove/\n pretrained_embeddings_urls = {}\n pretrained_embeddings_urls[\"glove.6B.50d.txt\"] = (\"http://nlp.stanford.edu/data/glove.6B.zip\", \"glove.6B.zip\")\n pretrained_embeddings_urls[\"glove.6B.100d.txt\"] = (\"http://nlp.stanford.edu/data/glove.6B.zip\", \"glove.6B.zip\")\n pretrained_embeddings_urls[\"glove.6B.200d.txt\"] = (\"http://nlp.stanford.edu/data/glove.6B.zip\", \"glove.6B.zip\")\n pretrained_embeddings_urls[\"glove.6B.300d.txt\"] = (\"http://nlp.stanford.edu/data/glove.6B.zip\", \"glove.6B.zip\")\n pretrained_embeddings_urls[\"glove.42B.300d.txt\"] = (\"http://nlp.stanford.edu/data/glove.42B.300d.zip\", \"glove.42B.300d.zip\")\n pretrained_embeddings_urls[\"glove.840B.300d.txt\"] = (\"http://nlp.stanford.edu/data/glove.840B.300d.zip\", \"glove.840B.300d.zip\")\n pretrained_embeddings_urls[\"glove.twitter.27B.txt\"] = (\"http://nlp.stanford.edu/data/glove.twitter.27B.zip\", \"glove.twitter.27B.zip\")\n pretrained_embeddings_urls[\"mimic.fastText.no_clean.300d.pickled\"] = (\"https://mednli.blob.core.windows.net/shared/word_embeddings/mimic.fastText.no_clean.300d.pickled\",\"mimic.fastText.no_clean.300d.pickled\")\n\n if (embeddings_name not in pretrained_embeddings_urls.keys()):\n logger.error(\"Cannot load the indicated pretrained embeddings (current '{}' must be one of {})\".format(embeddings_name, pretrained_embeddings_urls.keys()))\n exit(1)\n\n logger.info(\"Initializing embeddings in folder {}\".format(folder))\n\n num_loaded_embs = 0\n # Set random embeddings for words \"out of vocabulary\".\n # embeddings = np.zeros((len(word_to_ix), embeddings_size))\n embeddings = np.random.normal(scale=0.6, size=(len(word_to_ix), embeddings_size))\n\n # Open the embeddings file.\n if embeddings_name == \"mimic.fastText.no_clean.300d.pickled\":\n # Check if pickle exists.\n file_name = pretrained_embeddings_urls[embeddings_name][1]\n if not io.check_file_existence(folder, file_name):\n # Try to download the pickle.\n url = pretrained_embeddings_urls[embeddings_name][0]\n logger.info(\"Downloading file '{}' from {}\".format(file_name, url))\n io.download(folder, file_name, url)\n else:\n logger.info(\"File '{}' found in {}\".format(embeddings_name, folder))\n\n # Load word embeddings map.\n word_embedding_map = io.load_pickle(logger, os.path.join(folder, file_name))\n # Iterate over map and cherry pick the vectors that fit our vocabulary.\n for w, index in word_to_ix.items():\n if w in word_embedding_map:\n vector = word_embedding_map[w]\n assert (len(vector) == embeddings_size), \"Embeddings size must be equal to the size of pretrained embeddings!\"\n embeddings[index] = vector\n num_loaded_embs += 1\n\n else:\n\n # Check presence of the file.\n if not io.check_file_existence(folder, embeddings_name):\n # Download and extract wikitext zip.\n io.download_extract_zip_file(logger, folder, pretrained_embeddings_urls[embeddings_name][0], pretrained_embeddings_urls[embeddings_name][1])\n else: \n logger.info(\"File '{}' containing pretrained embeddings found in '{}' folder\".format(embeddings_name, folder))\n\n # Get number of lines/vectors.\n num_lines = sum([1 for line in open(os.path.join(folder, embeddings_name))])\n t = tqdm.tqdm(total=num_lines)\n\n with open(os.path.join(folder, embeddings_name)) as f:\n # Parse file and cherry pick the vectors that fit our vocabulary.\n for line in f.readlines():\n values = line.split()\n if len(values) > embeddings_size+1:\n #print(len(values))\n # Case: two (or more) words!\n num_words = len(values) - embeddings_size\n words = values[0:num_words]\n word = ' '.join(words)\n #print(word)\n # Get remaining vector.\n vector = np.array(values[num_words:], dtype='float32')\n else:\n # Get word.\n word = values[0]\n # Get remaining vector.\n vector = np.array(values[1:], dtype='float32')\n # Get index.\n index = word_to_ix.get(word)\n if index:\n assert (len(vector) == embeddings_size), \"Embeddings size must be equal to the size of pretrained embeddings!\"\n # Ok, set vector.\n embeddings[index] = vector\n # Increment counter.\n num_loaded_embs += 1\n t.update()\n t.close()\n\n logger.info(\"Loaded {} pretrained embeddings for vocabulary of size {} from {}\".format(num_loaded_embs, len(word_to_ix), embeddings_name))\n\n # Return matrix with embeddings.\n return torch.from_numpy(embeddings).float()\n" }, { "path": "ptp/components/mixins/io.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport sys\nimport numpy as np\nimport shutil\nimport zipfile\nimport time\nimport requests\nfrom pathlib import Path\nimport pickle\n\n\ndef load_pickle(logger, filename, encoding=\"ASCII\"):\n \"\"\"\n Loads picke from file.\n\n :param logger: Logger object.\n :param filename: Absolute path along with the name of the file to be loaded.\n :param encoding: file encoding (DEFAULT: \"ASCII\")\n \"\"\"\n try:\n with open(str(filename), 'rb') as f:\n obj = pickle.load(f, encoding=encoding)\n logger.info('Loaded: %s', filename)\n\n except EOFError:\n logger.warning('Cannot load: %s', filename)\n obj = None\n\n return obj\n\n\ndef save_nparray_to_csv_file(folder, filename, nparray, sep=','):\n \"\"\" \n Writes numpy array to csv file.\n\n :param folder: Relative path to to folder.\n :type folder: str\n\n :param filename: Name of the file.\n :type filename: str\n\n :param nparray: Numpy array.\n\n :param sep: Separator (DEFAULT: ',')\n \"\"\"\n # Make sure folder exists.\n os.makedirs(os.path.dirname(os.path.expanduser(folder) +'/'), exist_ok=True)\n\n name = os.path.join(os.path.expanduser(folder), filename)\n print(name)\n \n # Write array to file, separate elements with commas.\n nparray.tofile(name, sep=sep, format=\"%s\")\n\n\ndef load_nparray_from_csv_file(folder, filename, dtype=float, sep=','):\n \"\"\" \n Loads numpy array from csv file.\n\n :param folder: Relative path to to folder.\n :type folder: str\n\n :param filename: Name of the file.\n :type filename: str\n\n :param dtype: Type of the array to load (DEFAULT: float)\n\n :return: Numpy array.\n \"\"\"\n # Absolute pathname of the file.\n name = os.path.join(os.path.expanduser(folder), filename)\n \n # Load array from file\n nparray = np.fromfile(name, dtype, count=-1, sep=',')\n\n # Return it.\n return nparray\n\n\n\ndef save_string_list_to_txt_file(folder, filename, data):\n \"\"\" \n Writes list of strings to txt file.\n\n :param folder: Relative path to to folder.\n :type folder: str\n\n :param filename: Name of the file.\n :type filename: str\n\n :param data: List containing strings (sententes, words etc.).\n \"\"\"\n # Make sure folder exists.\n os.makedirs(os.path.dirname(os.path.expanduser(folder) +'/'), exist_ok=True)\n\n # Write elements in separate lines. \n with open(os.path.join(os.path.expanduser(folder), filename), mode='w+') as txtfile:\n txtfile.write('\\n'.join(data))\n\n\ndef load_string_list_from_txt_file(folder, filename):\n \"\"\"\n Loads data from txt file.\n\n :return: List of strings (e.g. list of sententes).\n \"\"\"\n data = []\n with open(os.path.join(os.path.expanduser(folder), filename), mode='rt') as txtfile:\n for line in txtfile:\n # Remove next line char.\n if line[-1] == '\\n':\n line = line[:-1]\n data.append(line)\n return data\n\n\ndef get_project_root() -> Path:\n \"\"\"\n Returns project root folder.\n \"\"\"\n return Path(__file__).parent.parent\n\n\ndef check_file_existence(folder, filename):\n \"\"\"\n Checks if file exists.\n\n :param folder: Relative path to to folder.\n :type folder: str\n\n :param filename: Name of the file.\n :type filename: str\n \"\"\"\n file_to_check = os.path.join(os.path.expanduser(folder), filename)\n if os.path.isdir(folder) and os.path.isfile(file_to_check):\n return True\n #self.logger.info('Dataset found at {}'.format(file_folder_to_check))\n else:\n return False\n\ndef check_files_existence(folder, filenames):\n \"\"\"\n Checks if all files in the list exist.\n\n :param folder: Relative path to to folder.\n :type folder: str\n\n :param filename: List of files\n :type lst: List of strings or a single string with filenames separated by spaces)\n \"\"\"\n # Check folder existence.\n if not os.path.isdir(folder):\n return False\n\n # Process list of files.\n if type(filenames) == str:\n filenames = filenames.split(\" \")\n\n # Check files one by one.\n for file in filenames:\n file_to_check = os.path.join(os.path.expanduser(folder), file)\n if not os.path.isfile(file_to_check):\n return False\n # Ok.\n return True\n \n\ndef download(folder, filename, url):\n \"\"\"\n Checks whether a file or folder exists at given path (relative to storage folder), \\\n otherwise downloads files from the given URL.\n\n :param folder: Relative path to to the folder.\n :type folder: str\n\n :param filename: Name of the file.\n :type filename: str\n\n :param url: URL to download the file from.\n :type url: str\n \"\"\"\n if check_file_existence(folder, filename):\n return\n \n # Make sure folder exists.\n os.makedirs(os.path.dirname(folder +'/'), exist_ok=True)\n\n # Download.\n file_result = os.path.join(os.path.expanduser(folder), filename)\n\n with open(os.path.expanduser(file_result), \"wb\") as f:\n global start_time\n start_time = time.time()\n with requests.get(url, stream=True) as r:\n content_length = int(r.headers.get('content-length', None))\n count = 0\n\n for chunk in r.iter_content(chunk_size=1024):\n if chunk:\n f.write(chunk)\n count += 1\n reporthook(count, 1024, content_length)\n\n #self.logger.info('Downloading {}'.format(url))\n\ndef reporthook(count, block_size, total_size):\n \"\"\"\n Progress bar function.\n \"\"\"\n global start_time\n if count == 0:\n start_time = time.time()\n return\n duration = time.time() - start_time\n progress_size = int(count * block_size)\n speed = int(progress_size / (1024 * duration))\n percent = int(count * block_size * 100 / total_size)\n sys.stdout.write(\"\\r...%d%%, %d MB, %d KB/s, %d seconds passed\" %\n (percent, progress_size / (1024 * 1024), speed, duration))\n sys.stdout.flush()\n\ndef download_extract_zip_file(logger, folder, url, zipfile_name):\n \"\"\"\n Method downloads zipfile from given URL and extracts it.\n\n :param logger: Logger object used for logging information during download.\n\n :param folder: Folder in which data will be stored.\n\n :param url: URL from which file will be downloaded.\n\n :param zipfile_name: Name of the zip file to be downloaded and extracted.\n\n \"\"\"\n logger.info(\"Initializing download in folder {}\".format(folder))\n\n if not check_file_existence(folder, zipfile_name):\n logger.info(\"Downloading file '{}' from {}\".format(zipfile_name, url))\n download(folder, zipfile_name, url)\n else:\n logger.info(\"File '{}' found in {}\".format(zipfile_name, folder))\n\n # Extract data from zip.\n logger.info(\"Extracting data from '{}'\".format(zipfile_name))\n with zipfile.ZipFile(os.path.join(folder, zipfile_name), 'r') as zip_ref:\n zip_ref.extractall(folder)\n \n logger.info(\"Download and extraciton finished\")\n\ndef move_files_between_dirs(logger, source_folder, dest_folder, filenames):\n \"\"\"\n Moves files between directories\n \"\"\"\n # Process list of files.\n if type(filenames) == str:\n filenames = filenames.split(\" \")\n\n for f in filenames:\n shutil.move(os.path.join(source_folder, f) , os.path.join(dest_folder, f))\n\n logger.info(\"Moved {} from {} to {}\".format(filenames, source_folder, dest_folder))\n\n" }, { "path": "ptp/components/mixins/word_mappings.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport csv\n\nclass WordMappings(object):\n \"\"\"\n Mixin class that handles the initialization of (word:index) mappings.\n Assumes that it is mixed-in into class that is derived from the component.\n .. warning::\n Constructor (__init__) of the Component class has to be called before component of the mixin WordMapping class.\n\n \"\"\"\n def __init__(self): #, name, class_type, config):\n \"\"\"\n Initializes the (word:index) mappings.\n\n Assumes that Component was initialized in advance, which means that the self object possesses the following objects:\n - self.config\n - self.globals\n - self.logger\n\n \"\"\"\n # Read the actual configuration.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Source and resulting (indexed) vocabulary.\n self.source_vocabulary_files = self.config['source_vocabulary_files']\n self.word_mappings_file = self.config['word_mappings_file']\n \n # Set aboslute path to file with word mappings.\n word_mappings_file_path = os.path.join(os.path.expanduser(self.data_folder), self.word_mappings_file)\n\n # Check if we want to export word mappings to globals.\n if self.config[\"import_word_mappings_from_globals\"]:\n self.word_to_ix = self.globals[\"word_mappings\"]\n assert (len(self.word_to_ix) > 0), \"The word mappings imported from global variables are empty!\"\n # We could also get vocabulary_size from globals... but what for;)\n\n elif self.word_mappings_file != \"\" and not self.config['regenerate']:\n if not os.path.exists(word_mappings_file_path):\n self.logger.warning(\"Cannot load word mappings from '{}' because the file does not exist\".format(word_mappings_file_path))\n # This is a show stopper.\n exit(-1)\n\n # Try to load the preprocessed word mappings.\n self.word_to_ix = load_word_mappings_from_csv_file(self.logger, self.data_folder, self.word_mappings_file)\n assert (len(self.word_to_ix) > 0), \"The word mappings loaded from file are empty!\"\n\n else:\n # Try to generate new word mappings from source files.\n self.word_to_ix = generate_word_mappings_from_source_files(self.logger, self.data_folder, self.source_vocabulary_files)\n assert (len(self.word_to_ix) > 0), \"The word mappings generated from sources are empty!\"\n # Ok, save mappings, so next time we will simply load them.\n save_word_mappings_to_csv_file(self.logger, self.data_folder, self.word_mappings_file, self.word_to_ix)\n\n # Check if additional tokens are present.\n self.additional_tokens = self.config[\"additional_tokens\"].split(',')\n for word in self.additional_tokens:\n # If new token.\n if word != '' and word not in self.word_to_ix:\n self.word_to_ix[word] = len(self.word_to_ix)\n\n if self.config[\"export_pad_index_to_globals\"]:\n self.globals[\"pad_index\"] = self.word_to_ix['']\n\n self.logger.info(\"Initialized word mappings with vocabulary of size {}\".format(len(self.word_to_ix)))\n\n # Check if we want to export word mappings to globals.\n if self.config[\"export_word_mappings_to_globals\"]:\n self.globals[\"word_mappings\"] = self.word_to_ix\n # Export vocabulary size to globals.\n self.globals[\"vocabulary_size\"] = len(self.word_to_ix)\n\n\n\ndef load_word_mappings_from_csv_file(logger, folder, filename):\n \"\"\"\n Loads (word:index) mappings from csv file.\n\n .. warning::\n There is an assumption that file will contain key:value pairs (no content checking for now!)\n\n :param logger: Logger object.\n\n :param folder: Relative path to to the folder.\n :type folder: str\n\n :param filename: File with encodings (absolute path + filename).\n\n :return: dictionary with word:index keys\n \"\"\" \n file_path = os.path.join(os.path.expanduser(folder), filename)\n\n if not os.path.exists(file_path):\n logger.warning(\"Cannot load word mappings from '{}' because the file does not exist\".format(file_path))\n\n with open(file_path, mode='rt') as csvfile:\n # Check the presence of the header.\n sniffer = csv.Sniffer()\n first_bytes = str(csvfile.read(256))\n has_header = sniffer.has_header(first_bytes)\n # Rewind.\n csvfile.seek(0) \n reader = csv.reader(csvfile)\n # Skip the header row.\n if has_header:\n next(reader) \n # Read the remaining rows.\n word_to_ix = {rows[0]:int(rows[1]) for rows in reader}\n\n logger.info(\"Loaded mappings of size {}\".format(len(word_to_ix)))\n return word_to_ix\n\n\ndef save_word_mappings_to_csv_file(logger, folder, filename, word_to_ix, fieldnames = [\"word\",\"index\"]):\n \"\"\"\n Saves (word:index) mappings dictionary to a file.\n\n :param logger: Logger object.\n\n :param folder: Relative path to to the folder.\n :type folder: str\n\n :param filename: Name of file with encodings.\n \n :param word_to_ix: Dictionary with word:index mappings to be saved.\n \n \"\"\"\n # Expand path.\n folder = os.path.expanduser(folder)\n # Make sure directory exists.\n os.makedirs(os.path.dirname(folder +'/'), exist_ok=True)\n\n file_path = os.path.join(folder, filename)\n\n with open(file_path, mode='w+') as csvfile:\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n # Create header.\n writer.writeheader()\n\n # Write word-index pairs.\n for (k,v) in word_to_ix.items():\n #print(\"{} : {}\".format(k,v))\n writer.writerow({fieldnames[0]:k, fieldnames[1]: v})\n\n logger.info(\"Saved mappings of size {} to file '{}'\".format(len(word_to_ix), file_path))\n\ndef pad_trunc_list(l: list, length: int, padding_value = 0, eos_value = None):\n \"\"\"\n Will apply padding / clipping to list to meet requested length.\n Works on the list in-place.\n\n :param l: List to manipulate\n\n :param length: Target length\n\n :param padding_value: Value to fill when padding. Default is int(0).\n\n :return: None\n \"\"\"\n if len(l) < length:\n if eos_value is not None:\n l.append(eos_value)\n l.extend([padding_value]*(length-len(l)))\n \n elif len(l) > length:\n #print(\"pad_trunc_list to cat!: {}\".format(len(l)))\n #exit(1)\n del l[length:]\n if eos_value is not None:\n l[length-1] = eos_value\n" }, { "path": "ptp/components/models/__init__.py", "content": "from .model import Model\n\n# General usage\nfrom .general_usage.feed_forward_network import FeedForwardNetwork\nfrom .general_usage.recurrent_neural_network import RecurrentNeuralNetwork\nfrom .general_usage.seq2seq import Seq2Seq\nfrom .general_usage.attention_decoder import AttentionDecoder\n\n# Language\nfrom .language.index_embeddings import IndexEmbeddings\nfrom .language.sentence_embeddings import SentenceEmbeddings\n\n# Vision\nfrom .vision.convnet_encoder import ConvNetEncoder\nfrom .vision.generic_image_encoder import GenericImageEncoder\nfrom .vision.lenet5 import LeNet5\n\n# Multi-modal reasoning\nfrom .multi_modal_reasoning.compact_bilinear_pooling import CompactBilinearPooling\nfrom .multi_modal_reasoning.factorized_bilinear_pooling import FactorizedBilinearPooling\nfrom .multi_modal_reasoning.low_rank_bilinear_pooling import LowRankBilinearPooling\nfrom .multi_modal_reasoning.question_driven_attention import QuestionDrivenAttention\nfrom .multi_modal_reasoning.relational_network import RelationalNetwork\nfrom .multi_modal_reasoning.self_attention import SelfAttention\n\n__all__ = [\n 'Model',\n\n # General usage\n 'FeedForwardNetwork',\n 'RecurrentNeuralNetwork',\n 'Seq2Seq',\n 'AttentionDecoder',\n\n # Language\n 'IndexEmbeddings',\n 'SentenceEmbeddings',\n\n # Vision\n 'ConvNetEncoder',\n 'GenericImageEncoder',\n 'LeNet5',\n\n # Multi-modal reasoning\n 'CompactBilinearPooling',\n 'FactorizedBilinearPooling',\n 'LowRankBilinearPooling',\n 'QuestionDrivenAttention',\n 'RelationalNetwork',\n 'SelfAttention'\n ]\n" }, { "path": "ptp/components/models/general_usage/attention_decoder.py", "content": "# Copyright (C) Alexis Asseman, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman\"\n\nimport torch\n\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass AttentionDecoder(Model): \n \"\"\"\n Single layer GRU decoder with attention:\n Bahdanau, D., Cho, K., & Bengio, Y. (2014). Neural machine translation by jointly learning to align and translate. arXiv preprint arXiv:1409.0473.\n \n Needs the full sequence of hidden states from the encoder as input, as well as the last hidden state from the encoder as input state.\n\n Code is based on https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n # Call constructors of parent classes.\n Model.__init__(self, name, AttentionDecoder, config)\n\n # Get input/output mode\n self.output_last_state = self.config[\"output_last_state\"]\n self.ffn_output = self.config[\"ffn_output\"]\n\n # Get prediction mode from configuration.\n self.prediction_mode = self.config[\"prediction_mode\"]\n if self.prediction_mode not in ['Dense','Last', 'None']:\n raise ConfigurationError(\"Invalid 'prediction_mode' (current {}, available {})\".format(self.prediction_mode, ['Dense','Last', 'None']))\n\n self.autoregression_length = self.config[\"autoregression_length\"]\n\n # Retrieve input size from global variables.\n self.key_input_size = self.global_keys[\"input_size\"]\n self.input_size = self.globals[\"input_size\"]\n if type(self.input_size) == list:\n if len(self.input_size) == 1:\n self.input_size = self.input_size[0]\n else:\n raise ConfigurationError(\"RNN input size '{}' must be a single dimension (current {})\".format(self.key_input_size, self.input_size))\n\n # Retrieve output (prediction) size from global params.\n self.prediction_size = self.globals[\"prediction_size\"]\n if type(self.prediction_size) == list:\n if len(self.prediction_size) == 1:\n self.prediction_size = self.prediction_size[0]\n else:\n raise ConfigurationError(\"RNN prediction size '{}' must be a single dimension (current {})\".format(self.key_prediction_size, self.prediction_size))\n \n # Retrieve hidden size from configuration.\n self.hidden_size = self.config[\"hidden_size\"]\n if type(self.hidden_size) == list:\n if len(self.hidden_size) == 1:\n self.hidden_size = self.hidden_size[0]\n else:\n raise ConfigurationError(\"RNN hidden_size must be a single dimension (current {})\".format(self.hidden_size))\n \n # Get dropout rate value from config.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create dropout layer.\n self.dropout = torch.nn.Dropout(dropout_rate)\n\n # Create rnn cell: hardcoded one layer GRU.\n self.rnn_cell = getattr(torch.nn, \"GRU\")(self.input_size, self.hidden_size, 1, dropout=dropout_rate, batch_first=True)\n\n # Create layers for the attention\n self.attn = torch.nn.Linear(self.hidden_size * 2, self.autoregression_length)\n self.attn_combine = torch.nn.Linear(self.hidden_size * 2, self.hidden_size)\n\n # Create the trainable initial input for the decoder (A trained token of sorts)\n self.sos_token = torch.zeros(1, self.input_size)\n torch.nn.init.xavier_uniform(self.sos_token)\n self.sos_token = torch.nn.Parameter(self.sos_token, requires_grad=True)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n self.key_input_state = self.stream_keys[\"input_state\"]\n if self.output_last_state:\n self.key_output_state = self.stream_keys[\"output_state\"]\n \n self.logger.info(\"Initializing RNN with input size = {}, hidden size = {} and prediction size = {}\".format(self.input_size, self.hidden_size, self.prediction_size))\n\n # Create the output layer.\n self.activation2output_layer = None\n if(self.ffn_output):\n self.activation2output_layer = torch.nn.Linear(self.hidden_size, self.prediction_size)\n \n # Create the final non-linearity.\n self.use_logsoftmax = self.config[\"use_logsoftmax\"]\n if self.use_logsoftmax:\n if self.prediction_mode == \"Dense\":\n # Used then returning dense prediction, i.e. every output of unfolded model.\n self.log_softmax = torch.nn.LogSoftmax(dim=2)\n else:\n # Used when returning only the last output.\n self.log_softmax = torch.nn.LogSoftmax(dim=1)\n\n def activation2output(self, activations):\n output = self.dropout(activations)\n\n if(self.ffn_output):\n #output = activations.squeeze(1)\n shape = activations.shape\n\n # Reshape to 2D tensor [BATCH_SIZE * SEQ_LEN x HIDDEN_SIZE]\n output = output.contiguous().view(-1, shape[2])\n\n # Propagate data through the output layer [BATCH_SIZE * SEQ_LEN x PREDICTION_SIZE]\n output = self.activation2output_layer(output)\n #output = output.unsqueeze(1)\n\n # Reshape back to 3D tensor [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\n output = output.view(shape[0], shape[1], output.size(1))\n\n return output\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n\n d[self.key_inputs] = DataDefinition([-1, -1, self.hidden_size], [torch.Tensor], \"Batch of encoder outputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\")\n\n # Input hidden state\n d[self.key_input_state] = DataDefinition([-1, 1, self.hidden_size], [torch.Tensor], \"Batch of RNN last hidden states passed from another RNN that will be used as initial [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n\n return d\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n \n if self.prediction_mode == \"Dense\":\n d[self.key_predictions] = DataDefinition([-1, -1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\")\n elif self.prediction_mode == \"Last\": # \"Last\"\n # Only last prediction.\n d[self.key_predictions] = DataDefinition([-1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\")\n\n # Output hidden state stream TODO: why do we need that?\n if self.output_last_state:\n d[self.key_output_state] = DataDefinition([-1, 1, self.hidden_size], [torch.Tensor], \"Batch of RNN final hidden states [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n \n return d\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass of the model.\n\n :param data_streams: DataStreams({'inputs', 'predictions ...}), where:\n\n - inputs: expected inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE],\n - predictions: returned output with predictions (log_probs) [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\n \"\"\"\n \n inputs = data_streams[self.key_inputs]\n batch_size = inputs.shape[0]\n #print(\"{}: input shape: {}, device: {}\\n\".format(self.name, inputs.shape, inputs.device))\n\n # Initialize hidden state from inputs - as last hidden state from external component.\n hidden = data_streams[self.key_input_state]\n # For RNNs (aside of LSTM): [BATCH_SIZE x NUM_LAYERS x HIDDEN_SIZE] -> [NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE]\n hidden = hidden.transpose(0,1)\n #print(\"{}: hidden shape: {}, device: {}\\n\".format(self.name, hidden.shape, hidden.device))\n\n # List that will contain the output sequence\n activations = []\n\n # First input to the decoder - trainable \"start of sequence\" token\n activations_partial = self.sos_token.expand(batch_size, -1).unsqueeze(1)\n\n # Feed back the outputs iteratively\n for i in range(self.autoregression_length):\n\n # Do the attention thing\n attn_weights = torch.nn.functional.softmax(\n self.attn(torch.cat((activations_partial.transpose(0, 1), hidden), 2)),\n dim=2\n )\n attn_applied = torch.bmm(attn_weights.transpose(0, 1), inputs)\n activations_partial = torch.cat((activations_partial, attn_applied), 2)\n activations_partial = self.attn_combine(activations_partial)\n activations_partial = torch.nn.functional.relu(activations_partial)\n\n # Feed through the RNN\n activations_partial, hidden = self.rnn_cell(activations_partial, hidden)\n activations_partial = self.activation2output(activations_partial)\n\n # Add the single step output into list\n if self.prediction_mode == \"Dense\":\n activations += [activations_partial]\n\n # Reassemble all the outputs from list into an output tensor\n if self.prediction_mode == \"Dense\":\n outputs = torch.cat(activations, 1)\n # Log softmax - along PREDICTION dim.\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n elif self.prediction_mode == \"Last\":\n if self.use_logsoftmax:\n outputs = self.log_softmax(activations_partial.squeeze(1))\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n\n # Output last hidden state, if requested\n if self.output_last_state:\n # For others: [NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE] -> [BATCH_SIZE x NUM_LAYERS x HIDDEN_SIZE] \n hidden = hidden.transpose(0,1)\n # Export last hidden state.\n data_streams.publish({self.key_output_state: hidden})\n" }, { "path": "ptp/components/models/general_usage/feed_forward_network.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass FeedForwardNetwork(Model): \n \"\"\"\n Simple model consisting of several stacked fully connected layers with ReLU non-linearities and dropout between them.\n Additionally, applies log softmax non-linearity to the output.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the classifier.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n # Call constructors of parent classes.\n Model.__init__(self, name, FeedForwardNetwork, config)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n\n self.dimensions = self.config[\"dimensions\"]\n\n # Retrieve input size from global variables.\n self.input_size = self.globals[\"input_size\"]\n if type(self.input_size) == list:\n if len(self.input_size) == 1:\n self.input_size = self.input_size[0]\n else:\n raise ConfigurationError(\"Input size '{}' must be a single dimension (current {})\".format(self.global_keys[\"input_size\"], self.input_size))\n\n # Retrieve output (prediction) size from global params.\n self.prediction_size = self.globals[\"prediction_size\"]\n if type(self.prediction_size) == list:\n if len(self.prediction_size) == 1:\n self.prediction_size = self.prediction_size[0]\n else:\n raise ConfigurationError(\"Prediction size '{}' must be a single dimension (current {})\".format(self.global_keys[\"prediction_size\"], self.prediction_size))\n \n self.logger.info(\"Initializing network with input size = {} and prediction size = {}\".format(self.input_size, self.prediction_size))\n\n # Create the module list.\n modules = []\n # Retrieve dropout rate value - if set, will put dropout between every layer.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Retrieve number of hidden layers, along with their sizes (numbers of hidden neurons from configuration).\n try:\n hidden_sizes = self.config[\"hidden_sizes\"]\n if type(hidden_sizes) == list:\n # Stack linear layers.\n input_dim = self.input_size\n for hidden_dim in hidden_sizes:\n # Add linear layer.\n modules.append( torch.nn.Linear(input_dim, hidden_dim) )\n # Add activation and dropout.\n modules.append( torch.nn.ReLU() )\n if (dropout_rate > 0):\n modules.append( torch.nn.Dropout(dropout_rate) )\n # Remember size.\n input_dim = hidden_dim\n\n # Add output layer.\n modules.append( torch.nn.Linear(input_dim, self.prediction_size) )\n\n self.logger.info(\"Created {} hidden layers\".format(len(hidden_sizes)))\n\n else:\n raise ConfigurationError(\"'hidden_sizes' must contain a list with numbers of neurons in hidden layers (currently {})\".format(self.hidden_sizes))\n\n except KeyError:\n # Not present, in that case create a simple classifier with 1 linear layer.\n modules.append( torch.nn.Linear(self.input_size, self.prediction_size) )\n \n # Create the final non-linearity.\n self.use_logsoftmax = self.config[\"use_logsoftmax\"]\n if self.use_logsoftmax:\n modules.append( torch.nn.LogSoftmax(dim=1) )\n\n # Finally create the sequential model out of those modules.\n self.layers = torch.nn.Sequential(*modules)\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition(([-1] * (self.dimensions -1)) + [self.input_size], [torch.Tensor], \"Batch of inputs, each represented as index [BATCH_SIZE x ... x INPUT_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_predictions: DataDefinition(([-1] * (self.dimensions -1)) + [self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x ... x PREDICTION_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass of the model.\n\n :param data_streams: DataStreams({'inputs', 'predictions ...}), where:\n\n - inputs: expected inputs [BATCH_SIZE x ... x INPUT_SIZE],\n - predictions: returned output with predictions (log_probs) [BATCH_SIZE x ... x NUM_CLASSES]\n \"\"\"\n # Get inputs.\n x = data_streams[self.key_inputs]\n \n #print(\"{}: input shape: {}, device: {}\\n\".format(self.name, x.shape, x.device))\n\n # Check that the input has the number of dimensions that we expect\n assert len(x.shape) == self.dimensions, \\\n \"Expected \" + str(self.dimensions) + \" dimensions for input, got \" + str(len(x.shape))\\\n + \" instead. Check number of dimensions in the config.\"\n\n # Reshape such that we do a broadcast over the last dimension\n origin_shape = x.shape\n x = x.contiguous().view(-1, origin_shape[-1])\n\n # Propagate inputs through all layers and activations.\n x = self.layers(x)\n\n # Restore the input dimensions but the last one (as it's been resized by the FFN)\n x = x.view(*origin_shape[0:self.dimensions-1], -1)\n\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: x})\n" }, { "path": "ptp/components/models/general_usage/recurrent_neural_network.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass RecurrentNeuralNetwork(Model): \n \"\"\"\n Simple Classifier consisting of fully connected layer with log softmax non-linearity.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n # Call constructors of parent classes.\n Model.__init__(self, name, RecurrentNeuralNetwork, config)\n\n # Get input mode from the configuration.\n self.input_mode = self.config[\"input_mode\"]\n if self.input_mode not in ['Dense','Autoregression_First', 'Autoregression_None']:\n raise ConfigurationError(\"Invalid 'input_mode' (current '{}', available {})\".format(self.input_mode, ['Dense','Autoregression_First', 'Autoregression_None']))\n\n # Get prediction mode from configuration.\n self.prediction_mode = self.config[\"prediction_mode\"]\n if self.prediction_mode not in ['Dense','Last', 'None']:\n raise ConfigurationError(\"Invalid 'prediction_mode' (current '{}', available {})\".format(self.prediction_mode, ['Dense','Last', 'None']))\n\n # Get source of initial hidden state from configuration.\n self.initial_state = self.config[\"initial_state\"]\n if self.initial_state not in [\"Zero\", \"Trainable\", \"Input\"]:\n raise ConfigurationError(\"Invalid 'initial_state' of the hidden state (current '{}', available {})\".format(self.initial_state, [\"Zero\", \"Trainable\", \"Input\"]))\n\n # Make sure that the input-output combination is valid.\n if self.prediction_mode == 'None' and 'Autoregression' in self.input_mode:\n raise ConfigurationError(\"Invalid combination of 'input_mode' and prediction_mode' (current '{}' and '{}')\".format(self.input_mode, self.prediction_mode))\n # TODO: Any others?\n\n # If we are returning any predictions, set up the right stream and variables.\n if self.prediction_mode != \"None\":\n self.key_predictions = self.stream_keys[\"predictions\"]\n # Retrieve output (prediction) size from global params.\n self.prediction_size = self.globals[\"prediction_size\"]\n # Check whether it is ok.\n if type(self.prediction_size) == list:\n if len(self.prediction_size) == 1:\n self.prediction_size = self.prediction_size[0]\n else:\n raise ConfigurationError(\"RNN prediction size '{}' must be a single dimension (current '{}')\".format(self.key_prediction_size, self.prediction_size))\n\n # If we are accepting any inputs, set up the right stream and variables.\n if \"None\" not in self.input_mode:\n # Retrieve stream key.\n self.key_inputs = self.stream_keys[\"inputs\"]\n # Retrieve input size from global variables.\n self.key_input_size = self.global_keys[\"input_size\"]\n self.input_size = self.globals[\"input_size\"]\n if type(self.input_size) == list:\n if len(self.input_size) == 1:\n self.input_size = self.input_size[0]\n else:\n raise ConfigurationError(\"RNN input size '{}' must be a single dimension (current {})\".format(self.key_input_size, self.input_size))\n else: \n # If there are no inputs, do we really need input_size.\n # Because it is autoregression mode and we can use prediction size instead.\n self.input_size = self.prediction_size\n\n # Setup options for autoregression.\n if \"Autoregression\" in self.input_mode:\n assert self.input_size == self.prediction_size, \"In autoregression mode, needs input_size == prediction_size.\"\n # Get max length from configuration.\n self.max_autoregression_length = self.config[\"max_autoregression_length\"]\n\n # Get number of layers from config.\n self.num_layers = self.config[\"num_layers\"]\n\n # Retrieve hidden size from configuration.\n self.hidden_size = self.config[\"hidden_size\"]\n if type(self.hidden_size) == list:\n if len(self.hidden_size) == 1:\n self.hidden_size = self.hidden_size[0]\n else:\n raise ConfigurationError(\"RNN hidden_size must be a single dimension (current {})\".format(self.hidden_size))\n \n # Get dropout rate value from config.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create RNN depending on the configuration\n self.cell_type = self.config[\"cell_type\"]\n if self.cell_type in ['LSTM', 'GRU']:\n # Create rnn cell.\n self.rnn_cell = getattr(torch.nn, self.cell_type)(self.input_size, self.hidden_size, self.num_layers, dropout=dropout_rate, batch_first=True)\n else:\n try:\n # Retrieve the non-linearity.\n nonlinearity = {'RNN_TANH': 'tanh', 'RNN_RELU': 'relu'}[self.cell_type]\n # Create rnn cell.\n self.rnn_cell = torch.nn.RNN(self.input_size, self.hidden_size, self.num_layers, nonlinearity=nonlinearity, dropout=dropout_rate, batch_first=True)\n except KeyError:\n raise ConfigurationError( \"Invalid RNN type, available options for 'cell_type' are ['LSTM', 'GRU', 'RNN_TANH', 'RNN_RELU'] (currently '{}')\".format(self.cell_type))\n \n # Parameters - for a single sample 2 x [NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE]\n h0 = torch.zeros(self.num_layers, 1, self.hidden_size)\n c0 = torch.zeros(self.num_layers, 1, self.hidden_size)\n\n # Check if initial state (h0/c0) is zero, trainable, or coming from input stream.\n self.init_hidden = None\n if self.initial_state == \"Trainable\":\n self.logger.info(\"Using trainable initial (h0/c0) state\")\n # Initialize a single vector used as hidden state.\n # Initialize it using xavier initialization.\n torch.nn.init.xavier_uniform(h0)\n # It will be trainable, i.e. the system will learn what should be the right initialization state.\n self.init_hidden = torch.nn.Parameter(h0, requires_grad=True)\n # Initilize memory cell in a similar way.\n if self.cell_type == 'LSTM':\n torch.nn.init.xavier_uniform(c0)\n self.init_memory = torch.nn.Parameter(c0, requires_grad=True)\n elif self.initial_state == \"Zero\":\n self.logger.info(\"Using zero initial (h0/c0) state\")\n # We will still embedd it into parameter to enable storing/loading of both types of models by each other.\n self.init_hidden = torch.nn.Parameter(h0, requires_grad=False)\n if self.cell_type == 'LSTM':\n self.init_memory = torch.nn.Parameter(c0, requires_grad=False)\n else: # \"Input\" means that it will be taken from the \"input_state\" stream.\n # Get adequate key mappings.\n self.key_input_state = self.stream_keys[\"input_state\"]\n self.logger.info(\"Will read initial (h0/c0) state from stream '{}\".format(self.key_input_state))\n\n\n # Setup for outputs.\n # Last state.\n self.output_last_state = self.config[\"output_last_state\"]\n if self.output_last_state:\n self.key_output_state = self.stream_keys[\"output_state\"]\n \n self.logger.info(\"Initializing RNN with input size = {}, hidden size = {} and prediction size = {}\".format(self.input_size, self.hidden_size, self.prediction_size))\n\n # Setup for the output layer (and associated non-linearities).\n self.use_output_layer = self.config[\"use_output_layer\"]\n if(self.use_output_layer):\n # Create dropout layer.\n self.dropout = torch.nn.Dropout(dropout_rate)\n # Create the layer.\n self.activation2output = torch.nn.Linear(self.hidden_size, self.prediction_size)\n \n # Setup for the final non-linearity.\n self.use_logsoftmax = self.config[\"use_logsoftmax\"]\n if self.use_logsoftmax:\n if self.prediction_mode == \"Dense\":\n # Used then returning dense prediction, i.e. every output of unfolded model.\n self.log_softmax = torch.nn.LogSoftmax(dim=2)\n else:\n # Used when returning only the last output.\n self.log_softmax = torch.nn.LogSoftmax(dim=1)\n\n\n def initialize_hiddens_state(self, batch_size):\n \"\"\"\n Function initializes hidden states, depending on the cell type.\n \"\"\"\n if self.cell_type == 'LSTM':\n # Return tuple (hidden_state, memory_cell).\n return (self.init_hidden.expand(self.num_layers, batch_size, self.hidden_size).contiguous(),\n self.init_memory.expand(self.num_layers, batch_size, self.hidden_size).contiguous() )\n else:\n # Return hidden_state.\n return self.init_hidden.expand(self.num_layers, batch_size, self.hidden_size).contiguous()\n\n\n def activation_to_output_pass(self, activations):\n \"\"\"\n Function propagates hidden state \"activations\" through output layer (that pass can be optionally turned off).\n \"\"\"\n if(self.use_output_layer):\n # Use dropout when using output layer.\n output = self.dropout(activations)\n\n #output = activations.squeeze(1)\n shape = activations.shape\n\n # Reshape to 2D tensor [BATCH_SIZE * SEQ_LEN x HIDDEN_SIZE]\n output = output.contiguous().view(-1, shape[2])\n\n # Propagate data through the output layer [BATCH_SIZE * SEQ_LEN x PREDICTION_SIZE]\n output = self.activation2output(output)\n #output = output.unsqueeze(1)\n\n # Reshape back to 3D tensor [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\n output = output.view(shape[0], shape[1], output.size(1))\n\n return output\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n # Input depending on the input_mode\n if self.input_mode == \"Dense\":\n d[self.key_inputs] = DataDefinition([-1, -1, self.input_size], [torch.Tensor], \"Batch of inputs, each being a sequence of items [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\")\n elif self.input_mode == \"Autoregression_First\":\n d[self.key_inputs] = DataDefinition([-1, self.input_size], [torch.Tensor], \"Batch of inputs, each being a single item [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\")\n #else: Autoregression_None: no inputs.\n\n # Input hidden state\n if self.initial_state == \"Input\":\n if self.cell_type == \"LSTM\":\n d[self.key_input_state] = DataDefinition([-1, 2, self.num_layers, self.hidden_size], [torch.Tensor], \"Batch of LSTM last hidden states (h0/c0) passed from another LSTM that will be used as initial [BATCH_SIZE x 2 x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n else:\n d[self.key_input_state] = DataDefinition([-1, self.num_layers, self.hidden_size], [torch.Tensor], \"Batch of RNN last hidden states passed from another RNN that will be used as initial [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n\n return d\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n \n # Output: predictions stream.\n if self.prediction_mode == \"Dense\":\n d[self.key_predictions] = DataDefinition([-1, -1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as sequence of probability distributions over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\")\n elif self.prediction_mode == \"Last\": # \"Last\"\n # Only last prediction.\n d[self.key_predictions] = DataDefinition([-1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as a single probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]\")\n # Else: no predictions.\n\n # Output: hidden state stream.\n if self.output_last_state:\n if self.cell_type == \"LSTM\":\n d[self.key_output_state] = DataDefinition([-1, 2, self.num_layers, self.hidden_size], [torch.Tensor], \"Batch of LSTM final hidden states (h0/c0) [BATCH_SIZE x 2 x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n else:\n d[self.key_output_state] = DataDefinition([-1, self.num_layers, self.hidden_size], [torch.Tensor], \"Batch of RNN final hidden states [BATCH_SIZE x NUM_LAYERS x SEQ_LEN x HIDDEN_SIZE]\")\n\n return d\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass of the model.\n\n :param data_streams: DataStreams({'inputs', 'predictions ...}), where:\n\n - inputs: expected inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE],\n - predictions: returned output with predictions (log_probs) [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\n \"\"\"\n \n inputs = None\n batch_size = None\n\n # Get inputs\n if \"None\" in self.input_mode:\n # Must be in autoregressive mode - retrieve batch_size from initial hidden state from encoder.\n batch_size = data_streams[self.key_input_state][0].shape[1]\n # Set zero inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE].\n inputs = torch.zeros(batch_size, self.hidden_size, requires_grad=False).type(self.app_state.FloatTensor)\n\n else:\n # Get inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\n inputs = data_streams[self.key_inputs]\n if inputs.dim() == 2:\n inputs = inputs.unsqueeze(1)\n batch_size = inputs.shape[0]\n #print(\"{}: input shape: {}, device: {}\\n\".format(self.name, inputs.shape, inputs.device))\n\n # Get initial state, depending on the settings.\n if self.initial_state == \"Input\":\n # Initialize hidden state from inputs - as last hidden state from external component.\n hidden = data_streams[self.key_input_state]\n # Flip batch and num_layer dims so batch will be third/second!\n if self.cell_type == 'LSTM':\n # For LSTM: [BATCH_SIZE x NUM_LAYERS x 2 x HIDDEN_SIZE] -> [2 x NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE]\n hidden = hidden.transpose(0,2)\n else: \n # For others: [BATCH_SIZE x NUM_LAYERS x HIDDEN_SIZE] -> [NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE]\n hidden = hidden.transpose(0,1)\n else:\n hidden = self.initialize_hiddens_state(batch_size)\n #print(\"{}: hidden shape: {}, device: {}\\n\".format(self.name, hidden.shape, hidden.device))\n\n activations = []\n\n # Check out operation mode.\n if \"Autoregression\" in self.input_mode: \n # Autoregressive mode - feed back outputs in the input\n activations_partial, hidden = self.rnn_cell(inputs, hidden)\n activations_partial = self.activation_to_output_pass(activations_partial)\n activations += [activations_partial]\n\n # Feed back the outputs iteratively\n for i in range(self.max_autoregression_length - 1):\n activations_partial, hidden = self.rnn_cell(activations_partial, hidden)\n activations_partial = self.activation_to_output_pass(activations_partial)\n # Add the single step output into list\n if self.prediction_mode == \"Dense\":\n activations += [activations_partial]\n # Reassemble all the outputs from list into an output sequence\n if self.prediction_mode == \"Dense\":\n outputs = torch.cat(activations, 1)\n # Log softmax - along PREDICTION dim.\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n elif self.prediction_mode == \"Last\":\n # Take only the last activations.\n outputs = activations_partial.squeeze(1)\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n\n else:\n # Normal mode - feed the entire input sequence at once\n activations, hidden = self.rnn_cell(inputs, hidden)\n\n if self.prediction_mode == \"Dense\":\n # Pass every activation through the output layer.\n outputs = self.activation_to_output_pass(activations)\n \n # Log softmax - along PREDICTION dim.\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n elif self.prediction_mode == \"Last\":\n outputs = self.activation_to_output_pass(activations.contiguous()[:, -1, :].unsqueeze(1))\n outputs = outputs.squeeze(1)\n\n # Log softmax - along PREDICTION dim.\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n \n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n elif self.prediction_mode == \"None\":\n # Nothing, since we don't want to keep the RNN's outputs\n pass\n\n if self.output_last_state:\n # Flip batch and num_layer dims so batch will be first!\n if self.cell_type == 'LSTM':\n # For LSTM: [2 x NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE] -> [BATCH_SIZE x NUM_LAYERS x 2 x HIDDEN_SIZE] \n hidden = hidden.transpose(0,2)\n else: \n # For others: [NUM_LAYERS x BATCH_SIZE x HIDDEN_SIZE] -> [BATCH_SIZE x NUM_LAYERS x HIDDEN_SIZE] \n hidden = hidden.transpose(0,1)\n # Export last hidden state.\n data_streams.publish({self.key_output_state: hidden})\n" }, { "path": "ptp/components/models/general_usage/seq2seq.py", "content": "# Copyright (C) aasseman, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman\"\n\nimport torch\n\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass Seq2Seq(Model): \n \"\"\"\n Simple Classifier consisting of fully connected layer with log softmax non-linearity.\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n # Call constructors of parent classes.\n Model.__init__(self, name, Seq2Seq, config)\n\n # Get input/output mode\n self.input_mode = self.config[\"input_mode\"]\n\n self.autoregression_length = self.config[\"autoregression_length\"]\n \n # Check if initial state (h0/c0) is zero, trainable, or coming from input stream.\n self.initial_state = self.config[\"initial_state\"]\n\n # Get number of layers from config.\n self.num_layers = self.config[\"num_layers\"]\n\n # Retrieve input size from global variables.\n self.key_input_size = self.global_keys[\"input_size\"]\n self.input_size = self.globals[\"input_size\"]\n if type(self.input_size) == list:\n if len(self.input_size) == 1:\n self.input_size = self.input_size[0]\n else:\n raise ConfigurationError(\"RNN input size '{}' must be a single dimension (current {})\".format(self.key_input_size, self.input_size))\n\n # Retrieve output (prediction) size from global params.\n self.prediction_size = self.globals[\"prediction_size\"]\n if type(self.prediction_size) == list:\n if len(self.prediction_size) == 1:\n self.prediction_size = self.prediction_size[0]\n else:\n raise ConfigurationError(\"RNN prediction size '{}' must be a single dimension (current {})\".format(self.key_prediction_size, self.prediction_size))\n \n # Retrieve hidden size from configuration.\n self.hidden_size = self.config[\"hidden_size\"]\n if type(self.hidden_size) == list:\n if len(self.hidden_size) == 1:\n self.hidden_size = self.hidden_size[0]\n else:\n raise ConfigurationError(\"RNN hidden_size must be a single dimension (current {})\".format(self.hidden_size))\n\n # Create RNN depending on the configuration\n self.cell_type = self.config[\"cell_type\"]\n if self.cell_type in ['LSTM', 'GRU']:\n # Create rnn cell.\n self.rnn_cell_enc = getattr(torch.nn, self.cell_type)(self.input_size, self.hidden_size, self.num_layers, batch_first=True)\n self.rnn_cell_dec = getattr(torch.nn, self.cell_type)(self.input_size, self.hidden_size, self.num_layers, batch_first=True)\n else:\n try:\n # Retrieve the non-linearity.\n nonlinearity = {'RNN_TANH': 'tanh', 'RNN_RELU': 'relu'}[self.cell_type]\n # Create rnn cell.\n self.rnn_cell_enc = torch.nn.RNN(self.input_size, self.hidden_size, self.num_layers, nonlinearity=nonlinearity, batch_first=True)\n self.rnn_cell_dec = torch.nn.RNN(self.input_size, self.hidden_size, self.num_layers, nonlinearity=nonlinearity, batch_first=True)\n except KeyError:\n raise ConfigurationError( \"Invalid RNN type, available options for 'cell_type' are ['LSTM', 'GRU', 'RNN_TANH', 'RNN_RELU'] (currently '{}')\".format(self.cell_type))\n \n # Parameters - for a single sample.\n h0 = torch.zeros(self.num_layers, 1, self.hidden_size)\n c0 = torch.zeros(self.num_layers, 1, self.hidden_size)\n\n self.init_hidden = None\n\n if self.initial_state == \"Trainable\":\n self.logger.info(\"Using trainable initial (h0/c0) state\")\n # Initialize a single vector used as hidden state.\n # Initialize it using xavier initialization.\n torch.nn.init.xavier_uniform(h0)\n # It will be trainable, i.e. the system will learn what should be the right initialization state.\n self.init_hidden = torch.nn.Parameter(h0, requires_grad=True)\n # Initilize memory cell in a similar way.\n if self.cell_type == 'LSTM':\n torch.nn.init.xavier_uniform(c0)\n self.init_memory = torch.nn.Parameter(c0, requires_grad=True)\n elif self.initial_state == \"Zero\":\n self.logger.info(\"Using zero initial (h0/c0) state\")\n # We will still embedd it into parameter to enable storing/loading of both types of models by each other.\n self.init_hidden = torch.nn.Parameter(h0, requires_grad=False)\n if self.cell_type == 'LSTM':\n self.init_memory = torch.nn.Parameter(c0, requires_grad=False)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n \n self.logger.info(\"Initializing RNN with input size = {}, hidden size = {} and prediction size = {}\".format(self.input_size, self.hidden_size, self.prediction_size))\n\n # Create the output layer.\n self.activation2output = torch.nn.Linear(self.hidden_size, self.prediction_size)\n \n # Create the final non-linearity.\n self.use_logsoftmax = self.config[\"use_logsoftmax\"]\n if self.use_logsoftmax:\n # Used then returning dense prediction, i.e. every output of unfolded model.\n self.log_softmax = torch.nn.LogSoftmax(dim=2)\n\n def initialize_hiddens_state(self, batch_size):\n\n if self.cell_type == 'LSTM':\n # Return tuple (hidden_state, memory_cell).\n return (self.init_hidden.expand(self.num_layers, batch_size, self.hidden_size).contiguous(),\n self.init_memory.expand(self.num_layers, batch_size, self.hidden_size).contiguous() )\n\n else:\n # Return hidden_state.\n return self.init_hidden.expand(self.num_layers, batch_size, self.hidden_size).contiguous()\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n\n d[self.key_inputs] = DataDefinition([-1, -1, self.input_size], [torch.Tensor], \"Batch of inputs, each represented as index [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\")\n\n return d\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n d = {}\n \n d[self.key_predictions] = DataDefinition([-1, -1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\")\n\n return d\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass of the model.\n\n :param data_streams: DataStreams({'inputs', 'predictions ...}), where:\n\n - inputs: expected inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE],\n - predictions: returned output with predictions (log_probs) [BATCH_SIZE x SEQ_LEN x PREDICTION_SIZE]\n \"\"\"\n \n # Get inputs [BATCH_SIZE x SEQ_LEN x INPUT_SIZE]\n inputs = data_streams[self.key_inputs]\n if inputs.dim() == 2:\n inputs = inputs.unsqueeze(1)\n batch_size = inputs.shape[0]\n\n\n # Initialize hidden state.\n hidden = self.initialize_hiddens_state(batch_size)\n\n\n # Encoder\n activations, hidden = self.rnn_cell_enc(inputs, hidden)\n activations_partial = self.activation2output(activations[:, -1, :])\n\n # Propagate inputs through rnn cell.\n activations_partial, hidden = self.rnn_cell_dec(activations_partial.unsqueeze(1), hidden)\n activations_partial = activations_partial.squeeze(1)\n activations_partial = self.activation2output(activations_partial)\n activations = [activations_partial]\n for i in range(self.autoregression_length - 1):\n activations_partial, hidden = self.rnn_cell_dec(activations_partial.unsqueeze(1), hidden)\n activations_partial = activations_partial.squeeze(1)\n activations_partial = self.activation2output(activations_partial)\n activations += [activations_partial]\n outputs = torch.stack(activations, 1)\n\n # Log softmax - along PREDICTION dim.\n if self.use_logsoftmax:\n outputs = self.log_softmax(outputs)\n\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: outputs})\n\n" }, { "path": "ptp/components/models/language/index_embeddings.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass IndexEmbeddings(Model):\n \"\"\"\n A simple embeddings layer.\n Embedds words using the provided indices of words.\n Assumes presence of a Word/Sencence Indexer.\n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the ``Embeddings`` layer.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n super(IndexEmbeddings, self).__init__(name, IndexEmbeddings, config)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve vocabulary size from globals.\n vocab_size = self.globals[\"vocab_size\"]\n\n # Retrieve embeddings size from configuration and export it to globals.\n self.embeddings_size = config['embeddings_size']\n self.globals[\"embeddings_size\"] = self.embeddings_size\n\n self.logger.info(\"Initializing embeddings layer with vocabulary size = {} and embeddings size = {}\".format(vocab_size, self.embeddings_size))\n\n # Finally: create the embeddings layer.\n self.embeddings = torch.nn.Embedding(vocab_size, self.embeddings_size)\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, -1], [torch.Tensor], \"Batch of of indexed sentences [BATCH_SIZE x SENTENCE_LENGTH]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, -1, self.embeddings_size], [torch.Tensor], \"Batch of embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]\")\n }\n\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass - performs embedding.\n\n :param data_streams: DataStreams({'images',**}), where:\n\n - inputs: expected indexed sentences [BATCH_SIZE x SENTENCE_LENGTH]\n - outputs: added embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]\n\n :type data_streams: ``miprometheus.utils.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n inputs = data_streams[self.key_inputs]\n\n # Embedd inputs.\n embeds = self.embeddings(inputs)\n\n # Add embeddings to datadict.\n data_streams.publish({self.key_outputs: embeds})\n" }, { "path": "ptp/components/models/language/sentence_embeddings.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.components.mixins.word_mappings import WordMappings, pad_trunc_list\nfrom ptp.data_types.data_definition import DataDefinition\n\nimport ptp.components.mixins.embeddings as emb\n\n\nclass SentenceEmbeddings(Model, WordMappings):\n \"\"\"\n Model responsible of embedding of whole sentences.\n\n For this purpose it:\n - creates its own vocabulary,\n - splits sentences into tokens (using NLTK tokenizer),\n - uses vocabulary to first transform tokens (one by one) to indices (first) and dense vectors (next).\n\n Optionally, it can load pretrained word embeddings (currently GloVe).\n\n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the ``SentenceEmbeddings`` layer.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n \"\"\"\n # Call base class constructors.\n Model.__init__(self, name, SentenceEmbeddings, config)\n WordMappings.__init__(self)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Force padding to a fixed length\n self.fixed_padding = self.config['fixed_padding']\n\n # Retrieve embeddings size from configuration and export it to globals.\n self.embeddings_size = self.config['embeddings_size']\n self.globals[\"embeddings_size\"] = self.embeddings_size\n\n # Create the embeddings layer.\n self.logger.info(\"Initializing embeddings layer with vocabulary size = {} and embeddings size = {}\".format(len(self.word_to_ix), self.embeddings_size))\n self.embeddings = torch.nn.Embedding(len(self.word_to_ix), self.embeddings_size, padding_idx=0) # Index of self.word_to_ix['']\n\n # Load the embeddings first.\n if self.config[\"pretrained_embeddings_file\"] != '':\n emb_vectors = emb.load_pretrained_embeddings(self.logger, self.data_folder, self.config[\"pretrained_embeddings_file\"], self.word_to_ix, self.embeddings_size)\n self.embeddings.weight = torch.nn.Parameter(emb_vectors)\n\n # Get index of from vocabulary.\n self.pad_index = self.word_to_ix['']\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, -1, 1], [list, list, str], \"Batch of sentences, each represented as a list of words [BATCH_SIZE] x [SEQ_LENGTH] x [string]\")\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, -1, self.embeddings_size], [torch.Tensor], \"Batch of embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]\")\n }\n\n\n def forward(self, data_streams):\n \"\"\"\n Forward pass - performs embedding.\n\n :param data_streams: DataStreams({'images',**}), where:\n\n - inputs: expected tokenized sentences [BATCH_SIZE x SENTENCE_LENGTH] x [string]\n - outputs: added embedded sentences [BATCH_SIZE x SENTENCE_LENGTH x EMBEDDING_SIZE]\n\n :type data_streams: ``miprometheus.utils.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n inputs = data_streams[self.key_inputs]\n #print(\"{}: input len: {}, device: {}\\n\".format(self.name, len(inputs), \"-\"))\n\n indices_list = []\n # Process samples 1 by one.\n for sample in inputs:\n assert isinstance(sample, (list,)), 'This embedder requires input sample to contain a list of words'\n # Process list.\n output_sample = []\n # Encode sample (list of words)\n for token in sample:\n # Get index.\n output_index = self.word_to_ix[token]\n # Add index to outputs.\n output_sample.append( output_index )\n\n # Apply fixed padding to all sequences if requested\n # Otherwise let torch.nn.utils.rnn.pad_sequence handle it and choose a dynamic padding\n if self.fixed_padding > 0:\n pad_trunc_list(output_sample, self.fixed_padding, padding_value=self.pad_index)\n\n #indices_list.append(self.app_state.FloatTensor(output_sample))\n indices_list.append(self.app_state.LongTensor(output_sample))\n\n # Padd indices using pad index retrieved from vocabulary.\n padded_indices = torch.nn.utils.rnn.pad_sequence(indices_list, batch_first=True, padding_value=self.pad_index)\n # Embedd indices.\n embedds = self.embeddings(padded_indices)\n \n #print(\"{}: embedds shape: {}, device: {}\\n\".format(self.name, embedds.shape, embedds.device))\n\n # Add embeddings to datadict.\n data_streams.publish({self.key_outputs: embedds})\n" }, { "path": "ptp/components/models/model.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta & Vincent Marois\"\n\nimport numpy as np\n\nfrom torch.nn import Module\n\nfrom ptp.components.component import Component\n\n\nclass Model(Module, Component):\n \"\"\"\n Class representing base class for all Models.\n\n Inherits from :py:class:`torch.nn.Module` as all subclasses will represent a trainable model.\n\n Hence, all subclasses should override the ``forward`` function.\n\n Implements features & attributes used by all subclasses.\n\n \"\"\"\n\n def __init__(self, name, class_type, config):\n \"\"\"\n Initializes a Model object.\n\n :param name: Model name.\n :type name: str\n\n :param class_type: Class type of the component.\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n This constructor:\n\n - calls base class constructors (save config, name, logger, app_state etc.)\n\n - initializes the best model loss (used to select which model to save) to ``np.inf``:\n\n >>> self.best_loss = np.inf\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, class_type, config)\n Module.__init__(self)\n\n # Flag indicating whether the model is frozen or not.\n self.frozen = False\n\n\n def save_to_checkpoint(self, chkpt):\n \"\"\"\n Adds model's state dictionary to checkpoint, which will be next stored to file.\n\n :param: Checkpoint (dictionary) that will be saved to file.\n \"\"\"\n chkpt[self.name] = self.state_dict()\n\n\n def load_from_checkpoint(self, chkpt, section=None):\n \"\"\"\n Loads state dictionary from checkpoint.\n\n :param chkpt: Checkpoint (dictionary) loaded from file.\n \n :param section: Name of the section containing params (DEFAULT: None, means that model name from current configuration will be used)\\\n \"\"\"\n if section is None:\n section = self.name\n self.load_state_dict(chkpt[section])\n\n def freeze(self):\n \"\"\"\n Freezes the trainable weigths of the model.\n \"\"\"\n # Freeze.\n self.frozen = True\n for param in self.parameters():\n param.requires_grad = False\n\n\n def summarize(self):\n \"\"\"\n Summarizes the model by showing the trainable/non-trainable parameters and weights\\\n per layer ( ``nn.Module`` ).\n\n Uses ``recursive_summarize`` to iterate through the nested structure of the model (e.g. for RNNs).\n\n :return: Summary as a str.\n\n \"\"\"\n # go recursively in the model architecture\n summary_str = self.recursive_summarize(self, 0, self.name)\n\n # Sum the model parameters.\n num_total_params = sum([np.prod(p.size()) for p in self.parameters()])\n mod_trainable_params = filter(lambda p: p.requires_grad, self.parameters())\n num_trainable_params = sum([np.prod(p.size()) for p in mod_trainable_params])\n\n summary_str += 'Total Trainable Params: {}\\n'.format(num_trainable_params)\n summary_str += 'Total Non-trainable Params: {}\\n'.format(num_total_params-num_trainable_params) \n summary_str += '='*80 + '\\n'\n\n return summary_str\n\n def recursive_summarize(self, module_, indent_, module_name_):\n \"\"\"\n Function that recursively inspects the (sub)modules and records their statistics\\\n (like names, types, parameters, their numbers etc.)\n\n :param module_: Module to be inspected.\n :type module_: ``nn.Module`` or subclass\n\n :param indent_: Current indentation level.\n :type indent_: int\n\n :param module_name_: Name of the module that will be displayed before its type.\n :type module_name_: str\n\n :return: Str summarizing the module.\n \"\"\"\n # Recursively inspect the children.\n child_lines = []\n for key, module in module_._modules.items():\n child_lines.append(self.recursive_summarize(module, indent_+1, key))\n\n # \"Leaf information\". \n mod_str = ''\n\n if indent_ > 0:\n mod_str += ' ' + '| ' * (indent_-1) + '+ '\n\n mod_str += module_name_ + \" (\" + module_._get_name() + ')'\n\n #print(\"Model's state_dict:\")\n #for param_tensor in module_.state_dict():\n # print(param_tensor, \"\\t\", module_.state_dict()[param_tensor].size())\n\n if indent_ == 0:\n if self.frozen:\n mod_str += \"\\t\\t[FROZEN]\"\n else:\n mod_str += \"\\t\\t[TRAINABLE]\"\n\n mod_str += '\\n'\n mod_str += ''.join(child_lines)\n\n # Get leaf weights and number of params - only for leafs!\n #if not child_lines:\n # Collect names and dimensions of all (named) params. \n mod_direct_params = list(filter(lambda np: np[0].find('.') == -1, module_.named_parameters()))\n if len(mod_direct_params) != 0:\n mod_weights = [(n, tuple(p.size())) for (n, p) in mod_direct_params]\n mod_str += ' ' + '| ' * (indent_) + '+ ' + 'Matrices: {}\\n'.format(mod_weights)\n\n # Sum the parameters.\n num_total_params = sum([np.prod(p.size()) for (_,p) in mod_direct_params])\n mod_trainable_params = filter(lambda np: np[1].requires_grad, mod_direct_params)\n num_trainable_params = sum([np.prod(p.size()) for (_,p) in mod_trainable_params])\n\n mod_str += ' ' + '| ' * (indent_) + ' Trainable Params: {}\\n'.format(num_trainable_params)\n mod_str += ' ' + '| ' * (indent_) + ' Non-trainable Params: {}\\n'.format(num_total_params - num_trainable_params)\n \n # Add line. \n mod_str += ' ' + '| ' * (indent_) + '\\n'\n \n return mod_str\n" }, { "path": "ptp/components/models/multi_modal_reasoning/compact_bilinear_pooling.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\nimport numpy as np\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass CompactBilinearPooling(Model):\n \"\"\"\n Element of one of classical baselines for Visual Question Answering.\n\n The model inputs (question and image encodings) are combined with Compact Bilinear Pooling mechanism.\n\n Fukui, A., Park, D. H., Yang, D., Rohrbach, A., Darrell, T., & Rohrbach, M. (2016). Multimodal compact bilinear pooling for visual question answering and visual grounding. arXiv preprint arXiv:1606.01847.\n\n Gao, Y., Beijbom, O., Zhang, N., & Darrell, T. (2016). Compact bilinear pooling. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 317-326).\n\n Inspired by implementation from:\n https://github.com/DeepInsight-PCALab/CompactBilinearPooling-Pytorch/blob/master/CompactBilinearPooling.py\n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(CompactBilinearPooling, self).__init__(name, CompactBilinearPooling, config)\n\n # Get key mappings.\n self.key_image_encodings = self.stream_keys[\"image_encodings\"]\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input/output sizes from globals.\n self.image_encoding_size = self.globals[\"image_encoding_size\"]\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n self.output_size = self.globals[\"output_size\"]\n\n # Initialize sketch projection matrices.\n image_sketch_projection_matrix = self.generate_count_sketch_projection_matrix(self.image_encoding_size, self.output_size)\n question_sketch_projection_matrix = self.generate_count_sketch_projection_matrix(self.question_encoding_size, self.output_size)\n\n # Make them parameters of the model, so can be stored/loaded and trained (optionally).\n trainable_projections = self.config[\"trainable_projections\"]\n self.image_sketch_projection_matrix = torch.nn.Parameter(image_sketch_projection_matrix, requires_grad=trainable_projections)\n self.question_sketch_projection_matrix = torch.nn.Parameter(question_sketch_projection_matrix, requires_grad=trainable_projections)\n\n\n def generate_count_sketch_projection_matrix(self, input_size, output_size):\n \"\"\" \n Initializes Count Sketch projection matrix for given input (size).\n Its role will be to project vector v∈Rn to y∈Rd.\n We initialize two vectors s∈{−1,1}n and h∈{1,...,d}n:\n * s contains either 1 or −1 for each index\n * h maps each index i in the input v to an index j in the output y.\n Both s and h are initialized randomly from a uniform distribution and remain constant.\n \"\"\"\n # Generate s: 1 or -1\n s = 2 * np.random.randint(2, size=input_size) - 1\n s = torch.from_numpy(s)\n #print(\"s=\",s.shape)\n\n # Generate h (indices)\n h = np.random.randint(output_size, size=input_size)\n #print(\"h=\",h.shape)\n indices = np.concatenate((np.arange(input_size)[..., np.newaxis],h[..., np.newaxis]), axis=1)\n indices = torch.from_numpy(indices)\n #print(\"indices=\",indices.shape)\n\n # Generate sparse matrix.\n sparse_sketch_matrix = torch.sparse.FloatTensor(indices.t(), s, torch.Size([input_size, output_size]))\n #print(\"\\n sparse_sketch_matrix=\",sparse_sketch_matrix.shape)\n # Return dense matrix.\n dense_ssm = sparse_sketch_matrix.to_dense().type(self.app_state.FloatTensor)\n #print(\"\\n dense_ssm=\",dense_ssm)\n\n return dense_ssm\n\n \n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_image_encodings: DataDefinition([-1, self.image_encoding_size], [torch.Tensor], \"Batch of encoded images [BATCH_SIZE x IMAGE_ENCODING_SIZE]\"),\n self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n # Unpack DataStreams.\n enc_img = data_streams[self.key_image_encodings]\n enc_q = data_streams[self.key_question_encodings]\n\n sketch_pm_img = self.image_sketch_projection_matrix\n sketch_pm_q = self.question_sketch_projection_matrix\n\n # Project both batches.\n sketch_img = enc_img.mm(sketch_pm_img)\n sketch_q = enc_q.mm(sketch_pm_q)\n\n # Add imaginary parts (with zeros).\n sketch_img_reim = torch.stack([sketch_img, torch.zeros(sketch_img.shape).type(self.app_state.FloatTensor)], dim=2)\n sketch_q_reim = torch.stack([sketch_q, torch.zeros(sketch_q.shape).type(self.app_state.FloatTensor)], dim=2)\n #print(\"\\n sketch_img_reim=\",sketch_img_reim)\n #print(\"\\n sketch_img_reim.shape=\",sketch_img_reim.shape)\n\n # Perform FFT.\n # Returns the real and the imaginary parts together as one tensor of the same shape of input.\n fft_img = torch.fft(sketch_img_reim, signal_ndim=1)\n fft_q = torch.fft(sketch_q_reim, signal_ndim=1)\n #print(fft_img)\n\n # Get real and imaginary parts.\n real1 = fft_img[:,:,0]\n imag1 = fft_img[:,:,1]\n real2 = fft_q[:,:,0]\n imag2 = fft_q[:,:,1]\n\n # Calculate product.\n fft_product = torch.stack([real1 * real2 - imag1 * imag2, real1 * imag2 + imag1 * real2], dim = -1)\n #print(\"fft_product=\",fft_product)\n\n # Inverse FFT.\n cbp = torch.ifft(fft_product, signal_ndim=1)[:,:,0]\n #print(\"cbp=\",cbp)\n\n # Add predictions to datadict.\n data_streams.publish({self.key_outputs: cbp})\n" }, { "path": "ptp/components/models/multi_modal_reasoning/factorized_bilinear_pooling.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Deepta Rajan\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\nimport torch.nn.functional as F\n\nclass FactorizedBilinearPooling(Model):\n \"\"\"\n Element of one of the classical baselines for Visual Question Answering.\n The multi-modal data is fused via sum-pooling of the element-wise multiplied high-dimensional representations and returned (for subsequent classification, done in a separate component e.g. ffn).\n\n On the basis of: Zhou Yu, Jun Yu. \"Beyond Bilinear: Generalized Multi-modal Factorized High-order Pooling for Visual Question Answering\" (2015).\n Code: https://github.com/Cadene/block.bootstrap.pytorch/blob/master/block/models/networks/fusions/fusions.py\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(FactorizedBilinearPooling, self).__init__(name, FactorizedBilinearPooling, config)\n\n # Get key mappings.\n self.key_image_encodings = self.stream_keys[\"image_encodings\"]\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input/output sizes from globals.\n self.image_encoding_size = self.globals[\"image_encoding_size\"]\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n\n # Get size of latent space and number of heads from config.\n self.latent_size = self.config[\"latent_size\"]\n self.factor = self.config[\"pool_factor\"]\n # Output feature size\n self.output_size = self.latent_size\n\n # Export to globals.\n self.globals[\"output_size\"] = self.output_size\n\n # Map image and question encodings to a common latent space of dimension 'latent_size'.\n self.image_encodings_ff = torch.nn.Linear(self.image_encoding_size, self.latent_size*self.factor)\n self.question_encodings_ff = torch.nn.Linear(self.question_encoding_size, self.latent_size*self.factor)\n\n # Create activation layer.\n self.activation = torch.nn.ReLU()\n\n # Retrieve dropout rate value - if set, will put dropout between every layer.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create dropout layer.\n self.dropout = torch.nn.Dropout(dropout_rate)\n\n\n def input_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_image_encodings: DataDefinition([-1, self.image_encoding_size], [torch.Tensor], \"Batch of encoded images [BATCH_SIZE x IMAGE_ENCODING_SIZE]\"),\n self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n enc_img = data_streams[self.key_image_encodings] #[48, 2048]\n enc_q = data_streams[self.key_question_encodings] #[48, 100]\n\n # Map image and question encodings to high-dimensional space using FF\n latent_img = self.dropout(self.image_encodings_ff(enc_img)) # [48, 512]\n latent_q = self.dropout(self.question_encodings_ff(enc_q)) # [48, 512]\n\n # Element-wise mutliplication of image and question encodings\n enc_z = latent_img * latent_q # [48, 512]\n\n # Dropout regularization\n enc_z = self.dropout(enc_z)\n enc_z = enc_z.view(enc_z.size(0), self.latent_size, self.factor) # [48, 256, 2]\n # Sum pooling\n enc_z = enc_z.sum(2) # [48, 256]\n # Power and L2 normalization\n enc_z = torch.sqrt(self.activation(enc_z)) - torch.sqrt(self.activation(-enc_z))\n outputs = F.normalize(enc_z, p=2, dim=1) # [48, 256]\n\n # Add predictions to datadict.\n data_streams.publish({self.key_outputs: outputs})\n" }, { "path": "ptp/components/models/multi_modal_reasoning/low_rank_bilinear_pooling.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass LowRankBilinearPooling(Model):\n \"\"\"\n Element of one of classical baselines for Visual Question Answering.\n The model inputs (question and image encodings) are fused via element-wise multiplication and returned (for subsequent classification, done in a separate component e.g. ffn).\n\n On the basis of: Jiasen Lu and Xiao Lin and Dhruv Batra and Devi Parikh. \"Deeper LSTM and normalized CNN visual question answering model\" (2015).\n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(LowRankBilinearPooling, self).__init__(name, LowRankBilinearPooling, config)\n\n # Get key mappings.\n self.key_image_encodings = self.stream_keys[\"image_encodings\"]\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input/output sizes from globals.\n self.image_encoding_size = self.globals[\"image_encoding_size\"]\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n self.output_size = self.globals[\"output_size\"]\n\n # Create the model.\n self.image_encodings_ff = torch.nn.Linear(self.image_encoding_size, self.output_size)\n self.question_encodings_ff = torch.nn.Linear(self.question_encoding_size, self.output_size)\n\n # Create activation layer.\n self.activation = torch.nn.ReLU()\n\n # Retrieve dropout rate value - if set, will put dropout between every layer.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create dropout layer.\n self.dropout = torch.nn.Dropout(dropout_rate)\n\n\n \n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_image_encodings: DataDefinition([-1, self.image_encoding_size], [torch.Tensor], \"Batch of encoded images [BATCH_SIZE x IMAGE_ENCODING_SIZE]\"),\n self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n enc_img = data_streams[self.key_image_encodings]\n enc_q = data_streams[self.key_question_encodings]\n\n # Apply nonlinearities and dropout on images.\n enc_img = self.activation(enc_img)\n enc_img = self.dropout(enc_img)\n\n # Apply nonlinearities and dropout on questions.\n enc_q = self.activation(enc_q)\n enc_q = self.dropout(enc_q)\n\n # Pass inputs layers mapping them to the same \"latent space\".\n latent_img = self.image_encodings_ff(enc_img)\n latent_q = self.question_encodings_ff(enc_q)\n \n # Element wise multiplication.\n outputs = latent_img * latent_q\n\n # Add predictions to datadict.\n data_streams.publish({self.key_outputs: outputs})\n" }, { "path": "ptp/components/models/multi_modal_reasoning/question_driven_attention.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Deepta Rajan\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass QuestionDrivenAttention(Model):\n \"\"\"\n Element of one of the classical baselines for Visual Question Answering.\n Attention-weighted image maps are computed based on the question.\n The multi-modal data (question and attention-weighted image maps) are fused via concatenation and returned (for subsequent classification, done in a separate component e.g. ffn).\n\n On the basis of: Vahid Kazemi Ali Elqursh. \"Show, Ask, Attend, and Answer: A Strong Baseline For Visual Question Answering\" (2017).\n Code: https://github.com/Cyanogenoid/pytorch-vqa/blob/master/model.py\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(QuestionDrivenAttention, self).__init__(name, QuestionDrivenAttention, config)\n\n # Get key mappings.\n self.key_feature_maps = self.stream_keys[\"feature_maps\"]\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input/output sizes from globals.\n self.feature_maps_height = self.globals[\"feature_maps_height\"]\n self.feature_maps_width = self.globals[\"feature_maps_width\"]\n self.feature_maps_depth = self.globals[\"feature_maps_depth\"]\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n\n # Get size of latent space and number of heads from config.\n self.latent_size = self.config[\"latent_size\"]\n self.num_attention_heads = self.config[\"num_attention_heads\"]\n\n # Output new attention weighted image encoding only, or both image and question image_encodings\n self.output_mode = self.config[\"output_mode\"]\n\n # Output feature size\n if(self.output_mode == 'Image'):\n self.output_size = self.feature_maps_depth*self.num_attention_heads\n elif(self.output_mode == 'Fusion'):\n self.output_size = self.feature_maps_depth*self.num_attention_heads + self.question_encoding_size\n else:\n print(\"'output_mode' unspecified for VQA Attention in config\") #TODO: find a better way to report corner case issue\n\n # Export to globals.\n self.globals[\"output_size\"] = self.output_size\n\n # Map image and question encodings to a common latent space of dimension 'latent_size'.\n self.image_encodings_conv = torch.nn.Conv2d(self.feature_maps_depth, self.latent_size, 1, bias=False)\n self.question_encodings_ff = torch.nn.Linear(self.question_encoding_size, self.latent_size)\n\n # Scalar-dot product attention function is implemented as a Conv operation\n self.attention_conv = torch.nn.Conv2d(self.latent_size, self.num_attention_heads, 1)\n\n # Create activation layer.\n self.activation = torch.nn.ReLU()\n\n # Retrieve dropout rate value - if set, will put dropout between every layer.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create dropout layer.\n self.dropout = torch.nn.Dropout(dropout_rate)\n\n\n def input_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_feature_maps: DataDefinition([-1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width], [torch.Tensor], \"Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]\"),\n self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n enc_img = data_streams[self.key_feature_maps] #[48, 2048, 7, 7]\n enc_q = data_streams[self.key_question_encodings] #[48, 100]\n # print(\"im_enc\", enc_img.shape)\n # print(\"enc_q\", enc_q.shape)\n\n # L2 norm of image encoding\n enc_img = enc_img / (enc_img.norm(p=2, dim=1, keepdim=True).expand_as(enc_img) + 1e-8)\n\n # Compute attention maps for image using questions\n latent_img = self.image_encodings_conv(self.dropout(enc_img)) # [48, 100, 7, 7]\n # print(\"latent_im\", latent_img.shape)\n latent_q = self.question_encodings_ff(self.dropout(enc_q)) # [48, 100]\n # print(\"latent_q\", latent_q.shape)\n latent_q_tile = tile_2d_over_nd(latent_q, latent_img) # [48, 100, 7, 7]\n # print(\"latent_q_tile\", latent_q_tile.shape)\n attention = self.activation(latent_img + latent_q_tile) #\n # print(\"attention\", attention.shape)\n attention = self.attention_conv(self.dropout(attention)) # [48, 2, 7, 7]\n # print(\"attention\", attention.shape)\n\n # Apply attention to image encoding\n attention_enc_img = apply_attention(enc_img, attention) # [48, 2048, 7, 7], [48, 2, 7, 7]\n # print(\"attention im\", attention_enc_img.shape)\n\n if(self.output_mode == 'Image'):\n # Output attention-weighted image encodings\n outputs = attention_enc_img\n elif(self.output_mode == 'Fusion'):\n # Fusion -- Concatenate attention-weighted image encodings and question encodings.\n outputs = torch.cat([attention_enc_img, enc_q], dim=1)\n # print(\"outputs\", outputs.shape)\n # Add predictions to datadict.\n data_streams.publish({self.key_outputs: outputs})\n\n\ndef tile_2d_over_nd(feature_vector, feature_map):\n \"\"\" Repeat the same feature vector over all spatial positions of a given feature map.\n The feature vector should have the same batch size and number of features as the feature map.\n \"\"\"\n n, c = feature_vector.size()\n spatial_size = feature_map.dim() - 2\n tiled = feature_vector.view(n, c, *([1] * spatial_size)).expand_as(feature_map)\n return tiled\n\n\ndef apply_attention(input, attention):\n \"\"\" Apply any number of attention maps over the input. \"\"\"\n n, c = input.size()[:2]\n glimpses = attention.size(1) # glimpses is equivalent to multiple heads in attention\n\n # flatten the spatial dims into the third dim, since we don't need to care about how they are arranged\n input = input.view(n, 1, c, -1) # [n, 1, c, s] [batch, 1, channels, height*width] [48, 1, 2048, 7*7]\n attention = attention.view(n, glimpses, -1) # [48, 2, 7*7]\n attention = torch.nn.functional.softmax(attention, dim=-1).unsqueeze(2) # [n, g, 1, s] [batch, multi_head, 1, height*width] [48, 2, 1, 7*7]\n weighted = attention * input # [n, g, c, s] [48, 2, 2048, 7*7]\n weighted_mean = weighted.sum(dim=-1) # [n, g, c] [48, 2, 2048]\n return weighted_mean.view(n, -1) # [48, 4096]\n" }, { "path": "ptp/components/models/multi_modal_reasoning/relational_network.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\n\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass RelationalNetwork(Model):\n \"\"\"\n Model implements relational network.\n Model expects image (CNN) features and encoded question.\n\n \n Santoro, A., Raposo, D., Barrett, D. G., Malinowski, M., Pascanu, R., Battaglia, P., & Lillicrap, T. (2017). A simple neural network module for relational reasoning. In Advances in neural information processing systems (pp. 4967-4976).\n Reference paper: https://arxiv.org/abs/1706.01427.\n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(RelationalNetwork, self).__init__(name, RelationalNetwork, config)\n\n # Get key mappings.\n self.key_feature_maps = self.stream_keys[\"feature_maps\"]\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input sizes from globals.\n self.feature_maps_height = self.globals[\"feature_maps_height\"]\n self.feature_maps_width = self.globals[\"feature_maps_width\"]\n self.feature_maps_depth = self.globals[\"feature_maps_depth\"]\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n \n # Create \"object\" coordinates.\n self.obj_coords = []\n for h in range(self.feature_maps_height):\n for w in range(self.feature_maps_width):\n self.obj_coords.append((h,w))\n\n # Calculate input size to the g_theta: two \"objects\" + question (+ optionally: image size)\n input_size = 2 * self.feature_maps_depth + self.question_encoding_size\n\n # Create the module list.\n modules = []\n\n # Retrieve dropout rate value - if set, will put dropout between every layer.\n dropout_rate = self.config[\"dropout_rate\"]\n\n # Create the model, i.e. the \"relational\" g_theta network.\n g_theta_sizes = self.config[\"g_theta_sizes\"]\n if type(g_theta_sizes) == list and len(g_theta_sizes) > 1:\n # First input dim.\n input_dim = input_size\n for hidden_dim in g_theta_sizes:\n # Add linear layer.\n modules.append( torch.nn.Linear(input_dim, hidden_dim) )\n # Add activation and dropout.\n modules.append( torch.nn.ReLU() )\n if (dropout_rate > 0):\n modules.append( torch.nn.Dropout(dropout_rate) )\n # Remember input dim of next layer.\n input_dim = hidden_dim\n\n # Add output layer.\n modules.append( torch.nn.Linear(input_dim, hidden_dim) )\n\n self.logger.info(\"Created g_theta network with {} layers\".format(len(g_theta_sizes)+1))\n\n else:\n raise ConfigurationError(\"'g_theta_sizes' must contain a list with numbers of neurons in g_theta layers (currently {})\".format(self.hidden_sizes))\n\n # Export output_size to globals.\n self.output_size = g_theta_sizes[-1]\n self.globals[\"output_size\"] = self.output_size\n\n # Finally create the sequential model out of those modules.\n self.g_theta = torch.nn.Sequential(*modules)\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_feature_maps: DataDefinition([-1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width], [torch.Tensor], \"Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]\"),\n self.key_question_encodings: DataDefinition([-1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n # Unpack DataStreams.\n feat_m = data_streams[self.key_feature_maps]\n enc_q = data_streams[self.key_question_encodings]\n\n # List [FEAT_WIDTH x FEAT_HEIGHT] of tensors [BATCH SIZE x (2 * FEAT_DEPTH + QUESTION_SIZE)]\n relational_inputs = []\n # Iterate through all pairs of \"objects\".\n for (h1,w1) in self.obj_coords:\n for (h2,w2) in self.obj_coords:\n # Get feature maps.\n fm1 = feat_m[:, :, h1,w1].view(-1, self.feature_maps_depth)\n fm2 = feat_m[:, :, h2,w2].view(-1, self.feature_maps_depth)\n # Concatenate with question [BATCH SIZE x (2 * FEAT_DEPTH + QUESTION_SIZE)]\n concat = torch.cat([fm1, fm2, enc_q], dim=1)\n relational_inputs.append(concat)\n\n # Stack tensors along with the batch dimension\n # [BATCH SIZE x (FEAT_WIDTH x FEAT_HEIGHT)^2 x (2 * FEAT_DEPTH + QUESTION_SIZE)]\n # i.e. [BATCH SIZE x NUM_RELATIONS x (2 * FEAT_DEPTH + QESTION_SIZE)]\n stacked_inputs = torch.stack(relational_inputs, dim=1)\n\n # Get shape [BATCH SIZE x (2 * FEAT_DEPTH + QESTION_SIZE)]\n shape = stacked_inputs.shape\n\n # Reshape such that we do a broadcast over the last dimension.\n stacked_inputs = stacked_inputs.contiguous().view(-1, shape[-1])\n\n # Pass it through g_theta.\n stacked_relations = self.g_theta(stacked_inputs)\n\n # Reshape to [BATCH_SIZE x NUM_RELATIONS x OUTPUT_SIZE]\n stacked_relations = stacked_relations.view(*shape[0:-1], self.output_size)\n\n # Element wise sum along relations [BATCH_SIZE x OUTPUT_SIZE]\n summed_relations = torch.sum(stacked_relations, dim=1)\n\n # Add outputs to datadict.\n data_streams.publish({self.key_outputs: summed_relations})\n" }, { "path": "ptp/components/models/multi_modal_reasoning/self_attention.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Deepta Rajan\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass SelfAttention(Model):\n \"\"\"\n Element of one of the classical baselines for Visual Question Answering.\n Attention within an image or text is computed.\n The attention weighted data (question or image) is returned (for subsequent classification, done in a separate component e.g. ffn).\n Currently only supports self-attention on text data\n\n On the basis of: Vaswani et. al Attention is all you need (2017)\n\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(SelfAttention, self).__init__(name, SelfAttention, config)\n\n # Get key mappings.\n self.key_question_encodings = self.stream_keys[\"question_encodings\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Retrieve input/output sizes from globals.\n self.question_encoding_size = self.globals[\"question_encoding_size\"]\n\n # Get size of latent space and number of heads from config.\n self.latent_size = self.config[\"latent_size\"]\n self.num_attention_heads = self.config[\"num_attention_heads\"]\n\n # Output feature size\n self.output_size = self.question_encoding_size*self.num_attention_heads\n # Create activation layer.\n self.activation = torch.nn.ReLU()\n\n # Create FF layers\n self.W1 = torch.nn.Linear(self.question_encoding_size, self.latent_size)\n self.W2 = torch.nn.Linear(self.latent_size, self.num_attention_heads)\n\n\n def input_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_question_encodings: DataDefinition([-1, -1, self.question_encoding_size], [torch.Tensor], \"Batch of encoded questions [BATCH_SIZE x SEQ_LEN x QUESTION_ENCODING_SIZE]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs [BATCH_SIZE x OUTPUT_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'images',**})\n :type data_streams: ``ptp.dadatypes.DataStreams``\n \"\"\"\n\n # Unpack DataStreams.\n input_enc = data_streams[self.key_question_encodings] # [batch, num_words, embed_dim] # Dense prediction from RNN\n batch_size = input_enc.size()[0] # [48, 8, 100]\n\n # Attention computed as two FF layers with ReLU activation and softmax for probabilities ==> softmax(FF(ReLU(FF(input))))\n self.Attention = torch.softmax(self.W2(self.activation(self.W1(input_enc))), dim = 1) # [48, 8, 4] [batch, num_words, num_heads]\n\n # Multiply attention weights with question encoding\n input_enc_weighted = torch.matmul(self.Attention.transpose(1,2),input_enc) # [48, 4, 100] [batch, num_heads, embed_dim]\n\n # Concatenate features from multi-head attention\n outputs = input_enc_weighted.view(batch_size, -1) # [48, 400] [batch, num_heads*embed_dim]\n # # Alternatively: combine multi-head attention using a mean or sum operation\n # outputs = torch.sum(input_enc_weighted,1)/self.num_attention_heads\n\n # Add predictions to datadict.\n data_streams.publish({self.key_outputs: outputs})\n" }, { "path": "ptp/components/models/vision/convnet_encoder.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Younes Bouhadjar, Vincent Marois, Tomasz Kornuta\"\n\nimport torch\nimport numpy as np\nimport torch.nn as nn\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass ConvNetEncoder(Model):\n \"\"\"\n A simple image encoder consisting of 3 consecutive convolutional layers. \\\n The parameters of input image (width, height and depth) are not hardcoded so the encoder can be adjusted for given application.\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Constructor of the ``SimpleConvNet``. \\\n\n The overall structure of this CNN is as follows:\n\n (Conv1 -> MaxPool1 -> ReLu) -> (Conv2 -> MaxPool2 -> ReLu) -> (Conv3 -> MaxPool3 -> ReLu)\n\n The parameters that the user can change are:\n\n - For Conv1, Conv2 & Conv3: number of output channels, kernel size, stride and padding.\n - For MaxPool1, MaxPool2 & MaxPool3: Kernel size\n\n\n .. note::\n\n We are using the default values of ``dilatation``, ``groups`` & ``bias`` for ``nn.Conv2D``.\n\n Similarly for the ``stride``, ``padding``, ``dilatation``, ``return_indices`` & ``ceil_mode`` of \\\n ``nn.MaxPool2D``.\n\n\n :param name: Name of the model (tken from the configuration file).\n\n :param config: dict of parameters (read from configuration ``.yaml`` file).\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n # Call base constructor.\n super(ConvNetEncoder, self).__init__(name, ConvNetEncoder, config)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_feature_maps = self.stream_keys[\"feature_maps\"]\n\n # Get input image information from the global parameters.\n self.input_width = self.globals[\"input_width\"]\n self.input_height = self.globals[\"input_height\"] \n self.input_depth = self.globals[\"input_depth\"]\n\n # Retrieve the Conv1 parameters.\n self.out_channels_conv1 = config['conv1']['out_channels']\n self.kernel_size_conv1 = config['conv1']['kernel_size']\n self.stride_conv1 = config['conv1']['stride']\n self.padding_conv1 = config['conv1']['padding']\n\n # Retrieve the MaxPool1 parameter.\n self.kernel_size_maxpool1 = config['maxpool1']['kernel_size']\n\n # Retrieve the Conv2 parameters.\n self.out_channels_conv2 = config['conv2']['out_channels']\n self.kernel_size_conv2 = config['conv2']['kernel_size']\n self.stride_conv2 = config['conv2']['stride']\n self.padding_conv2 = config['conv2']['padding']\n\n # Retrieve the MaxPool2 parameter.\n self.kernel_size_maxpool2 = config['maxpool2']['kernel_size']\n\n # Retrieve the Conv3 parameters.\n self.out_channels_conv3 = config['conv3']['out_channels']\n self.kernel_size_conv3 = config['conv3']['kernel_size']\n self.stride_conv3 = config['conv3']['stride']\n self.padding_conv3 = config['conv3']['padding']\n\n # Retrieve the MaxPool3 parameter.\n self.kernel_size_maxpool3 = config['maxpool3']['kernel_size']\n\n # We can compute the spatial size of the output volume as a function of the input volume size (W),\n # the receptive field size of the Conv Layer neurons (F), the stride with which they are applied (S),\n # and the amount of zero padding used (P) on the border.\n # The corresponding equation is conv_size = ((W−F+2P)/S)+1.\n\n # doc for nn.Conv2D: https://pytorch.org/docs/stable/nn.html#torch.nn.Conv2d\n # doc for nn.MaxPool2D: https://pytorch.org/docs/stable/nn.html#torch.nn.MaxPool2d\n\n # ----------------------------------------------------\n # Conv1\n self.conv1 = nn.Conv2d(in_channels=self.input_depth,\n out_channels=self.out_channels_conv1,\n kernel_size=self.kernel_size_conv1,\n stride=self.stride_conv1,\n padding=self.padding_conv1,\n dilation=1,\n groups=1,\n bias=True)\n\n self.width_features_conv1 = np.floor(\n ((self.input_width - self.kernel_size_conv1 + 2*self.padding_conv1) / self.stride_conv1) + 1)\n self.height_features_conv1 = np.floor(\n ((self.input_height - self.kernel_size_conv1 + 2*self.padding_conv1) / self.stride_conv1) + 1)\n\n # ----------------------------------------------------\n # MaxPool1\n self.maxpool1 = nn.MaxPool2d(kernel_size=self.kernel_size_maxpool1)\n\n self.width_features_maxpool1 = np.floor(\n ((self.width_features_conv1 - self.maxpool1.kernel_size + 2 * self.maxpool1.padding) / self.maxpool1.stride) + 1)\n\n self.height_features_maxpool1 = np.floor(\n ((self.height_features_conv1 - self.maxpool1.kernel_size + 2 * self.maxpool1.padding) / self.maxpool1.stride) + 1)\n\n # ----------------------------------------------------\n # Conv2\n self.conv2 = nn.Conv2d(in_channels=self.out_channels_conv1,\n out_channels=self.out_channels_conv2,\n kernel_size=self.kernel_size_conv2,\n stride=self.stride_conv2,\n padding=self.padding_conv2,\n dilation=1,\n groups=1,\n bias=True)\n\n self.width_features_conv2 = np.floor(\n ((self.width_features_maxpool1 - self.kernel_size_conv2 + 2*self.padding_conv2) / self.stride_conv2) + 1)\n self.height_features_conv2 = np.floor(\n ((self.height_features_maxpool1 - self.kernel_size_conv2 + 2*self.padding_conv2) / self.stride_conv2) + 1)\n\n # ----------------------------------------------------\n # MaxPool2\n self.maxpool2 = nn.MaxPool2d(kernel_size=self.kernel_size_maxpool2)\n\n self.width_features_maxpool2 = np.floor(\n ((self.width_features_conv2 - self.maxpool2.kernel_size + 2 * self.maxpool2.padding) / self.maxpool2.stride) + 1)\n self.height_features_maxpool2 = np.floor(\n ((self.height_features_conv2 - self.maxpool2.kernel_size + 2 * self.maxpool2.padding) / self.maxpool2.stride) + 1)\n\n # ----------------------------------------------------\n # Conv3\n self.conv3 = nn.Conv2d(in_channels=self.out_channels_conv2,\n out_channels=self.out_channels_conv3,\n kernel_size=self.kernel_size_conv3,\n stride=self.stride_conv3,\n padding=self.padding_conv3,\n dilation=1,\n groups=1,\n bias=True)\n\n self.width_features_conv3 = np.floor(\n ((self.width_features_maxpool2 - self.kernel_size_conv3 + 2*self.padding_conv3) / self.stride_conv3) + 1)\n self.height_features_conv3 = np.floor(\n ((self.height_features_maxpool2 - self.kernel_size_conv3 + 2*self.padding_conv3) / self.stride_conv3) + 1)\n\n # ----------------------------------------------------\n # MaxPool3\n self.maxpool3 = nn.MaxPool2d(kernel_size=self.kernel_size_maxpool3)\n\n self.width_features_maxpool3 = np.floor(\n ((self.width_features_conv3 - self.maxpool3.kernel_size + 2 * self.maxpool3.padding) / self.maxpool3.stride) + 1)\n\n self.height_features_maxpool3 = np.floor(\n ((self.height_features_conv3 - self.maxpool1.kernel_size + 2 * self.maxpool3.padding) / self.maxpool3.stride) + 1)\n\n # Set global variables: output dims\n self.globals[\"feature_map_height\"] = self.height_features_maxpool3\n self.globals[\"feature_map_width\"] = self.width_features_maxpool3\n self.globals[\"feature_map_depth\"] = self.out_channels_conv3\n \n # log some info.\n self.logger.info('Input: [-1, {}, {}, {}]'.format(self.input_depth, self.input_width, self.input_height))\n self.logger.info('Computed output shape of each layer:')\n self.logger.info('Conv1: [-1, {}, {}, {}]'.format(self.out_channels_conv1, self.width_features_conv1,\n self.height_features_conv1))\n self.logger.info('MaxPool1: [-1, {}, {}, {}]'.format(self.out_channels_conv1, self.width_features_maxpool1,\n self.height_features_maxpool1))\n self.logger.info('Conv2: [-1, {}, {}, {}]'.format(self.out_channels_conv2, self.width_features_conv2,\n self.height_features_conv2))\n self.logger.info('MaxPool2: [-1, {}, {}, {}]'.format(self.out_channels_conv2, self.width_features_maxpool2,\n self.height_features_maxpool2))\n self.logger.info('Conv3: [-1, {}, {}, {}]'.format(self.out_channels_conv3, self.width_features_conv3,\n self.height_features_conv3))\n self.logger.info('MaxPool3: [-1, {}, {}, {}]'.format(self.out_channels_conv3, self.width_features_maxpool3,\n self.height_features_maxpool3))\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, self.input_depth, self.input_height, self.input_width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE WIDTH]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_feature_maps: DataDefinition([-1, self.out_channels_conv3, self.height_features_maxpool3, self.width_features_maxpool3], [torch.Tensor], \"Batch of filter maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n forward pass of the ``SimpleConvNet`` model.\n\n :param data_streams: DataStreams({'inputs','outputs'}), where:\n\n - inputs: [batch_size, in_depth, in_height, in_width],\n - feature_maps: batch of feature maps [batch_size, out_depth, out_height, out_width]\n\n \"\"\"\n # get images\n images = data_streams[self.key_inputs]\n\n # apply Convolutional layer 1\n out_conv1 = self.conv1(images)\n\n # apply max_pooling and relu\n out_maxpool1 = torch.nn.functional.relu(self.maxpool1(out_conv1))\n\n # apply Convolutional layer 2\n out_conv2 = self.conv2(out_maxpool1)\n\n # apply max_pooling and relu\n out_maxpool2 = torch.nn.functional.relu(self.maxpool2(out_conv2))\n\n # apply Convolutional layer 3\n out_conv3 = self.conv3(out_maxpool2)\n\n # apply max_pooling and relu\n out_maxpool3 = torch.nn.functional.relu(self.maxpool3(out_conv3))\n\n # Add output to datadict.\n data_streams.publish({self.key_feature_maps: out_maxpool3})\n" }, { "path": "ptp/components/models/vision/generic_image_encoder.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nimport torch\nimport torchvision.models as models\n\nfrom ptp.configuration.config_parsing import get_value_from_dictionary\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass GenericImageEncoder(Model):\n \"\"\"\n Class\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the ``LeNet5`` model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(GenericImageEncoder, self).__init__(name, GenericImageEncoder, config)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Get operation modes.\n self.return_feature_maps = self.config[\"return_feature_maps\"]\n pretrained = self.config[\"pretrained\"]\n\n # Get model type from configuration.\n self.model_type = get_value_from_dictionary(\"model_type\", self.config, \"vgg16 | densenet121 | resnet152 | resnet50\".split(\" | \"))\n\n if(self.model_type == 'vgg16'):\n # Get VGG16\n self.model = models.vgg16(pretrained=pretrained)\n\n if self.return_feature_maps:\n # Use only the \"feature encoder\".\n self.model = self.model.features\n\n # Height of the returned features tensor (SET)\n self.feature_maps_height = 7\n self.globals[\"feature_maps_height\"] = self.feature_maps_height\n # Width of the returned features tensor (SET)\n self.feature_maps_width = 7\n self.globals[\"feature_maps_width\"] = self.feature_maps_width\n # Depth of the returned features tensor (SET)\n self.feature_maps_depth = 512\n self.globals[\"feature_maps_depth\"] = self.feature_maps_depth\n\n else:\n # Use the whole model, but cut/reshape only the last layer.\n self.output_size = self.globals[\"output_size\"]\n # \"Replace\" the last layer.\n self.model.classifier._modules['6'] = torch.nn.Linear(4096, self.output_size)\n\n elif(self.model_type == 'densenet121'):\n # Get densenet121\n self.model = models.densenet121(pretrained=pretrained)\n\n if self.return_feature_maps:\n raise ConfigurationError(\"'densenet121' doesn't support 'return_feature_maps' mode (yet)\")\n\n # Use the whole model, but cut/reshape only the last layer.\n self.output_size = self.globals[\"output_size\"]\n self.model.classifier = torch.nn.Linear(1024, self.output_size)\n\n\n elif(self.model_type == 'resnet152'):\n # Get resnet152\n self.model = models.resnet152(pretrained=pretrained)\n\n if self.return_feature_maps:\n # Get all modules exluding last (avgpool) and (fc)\n modules=list(self.model.children())[:-2]\n self.model=torch.nn.Sequential(*modules) \n\n # Height of the returned features tensor (SET)\n self.feature_maps_height = 7\n self.globals[\"feature_maps_height\"] = self.feature_maps_height\n # Width of the returned features tensor (SET)\n self.feature_maps_width = 7\n self.globals[\"feature_maps_width\"] = self.feature_maps_width\n # Depth of the returned features tensor (SET)\n self.feature_maps_depth = 2048\n self.globals[\"feature_maps_depth\"] = self.feature_maps_depth\n\n else:\n # Use the whole model, but cut/reshape only the last layer.\n self.output_size = self.globals[\"output_size\"]\n self.model.fc = torch.nn.Linear(2048, self.output_size)\n\n elif(self.model_type == 'resnet50'):\n # Get resnet50\n self.model = models.resnet50(pretrained=pretrained)\n\n if self.return_feature_maps:\n # Get all modules exluding last (avgpool) and (fc)\n modules=list(self.model.children())[:-2]\n self.model=torch.nn.Sequential(*modules) \n\n # Height of the returned features tensor (SET)\n self.feature_maps_height = 7\n self.globals[\"feature_maps_height\"] = self.feature_maps_height\n # Width of the returned features tensor (SET)\n self.feature_maps_width = 7\n self.globals[\"feature_maps_width\"] = self.feature_maps_width\n # Depth of the returned features tensor (SET)\n self.feature_maps_depth = 2048\n self.globals[\"feature_maps_depth\"] = self.feature_maps_depth\n\n else:\n # Use the whole model, but cut/reshape only the last layer.\n self.output_size = self.globals[\"output_size\"]\n self.model.fc = torch.nn.Linear(2048, self.output_size)\n\n\n def input_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, 3, 224, 224], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE WIDTH]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n if self.return_feature_maps:\n return {\n self.key_outputs: DataDefinition([-1, self.feature_maps_depth, self.feature_maps_height, self.feature_maps_width], [torch.Tensor], \"Batch of feature maps [BATCH_SIZE x FEAT_DEPTH x FEAT_HEIGHT x FEAT_WIDTH]\")\n }\n else:\n return {\n self.key_outputs: DataDefinition([-1, self.output_size], [torch.Tensor], \"Batch of outputs, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the model.\n\n :param data_streams: DataStreams({'inputs', ....}), where:\n\n - inputs: expected stream containing images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE WIDTH]\n - outpus: added stream containing outputs [BATCH_SIZE x PREDICTION_SIZE]\n\n :type data_streams: ``ptp.data_types.DataStreams``\n\n \"\"\"\n # Unpack DataStreams.\n img = data_streams[self.key_inputs]\n\n #print(\"{}: input shape: {}, device: {}\\n\".format(self.name, img.shape, img.device))\n\n outputs = self.model(img)\n\n # Add outputs to datadict.\n data_streams.publish({self.key_outputs: outputs})\n" }, { "path": "ptp/components/models/vision/lenet5.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta & Vincent Marois\"\n\n\nimport torch\n\nfrom ptp.components.models.model import Model\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass LeNet5(Model):\n \"\"\"\n A classical LeNet-5 model for MNIST digits classification. \n \"\"\" \n def __init__(self, name, config):\n \"\"\"\n Initializes the ``LeNet5`` model, creates the required layers.\n\n :param name: Name of the model (taken from the configuration file).\n\n :param config: Parameters read from configuration file.\n :type config: ``ptp.configuration.ConfigInterface``\n\n \"\"\"\n super(LeNet5, self).__init__(name, LeNet5, config)\n\n # Get key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n\n # Retrieve prediction size from globals.\n self.prediction_size = self.globals[\"prediction_size\"]\n\n # Create the LeNet-5 layers.\n self.conv1 = torch.nn.Conv2d(1, 6, kernel_size=(5, 5))\n self.maxpool1 = torch.nn.MaxPool2d(kernel_size=(2, 2), stride=2)\n self.conv2 = torch.nn.Conv2d(6, 16, kernel_size=(5, 5))\n self.maxpool2 = torch.nn.MaxPool2d(kernel_size=(2, 2), stride=2)\n self.conv3 = torch.nn.Conv2d(16, 120, kernel_size=(5, 5))\n self.linear1 = torch.nn.Linear(120, 84)\n self.linear2 = torch.nn.Linear(84, self.prediction_size)\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition([-1, 1, 32, 32], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE WIDTH]\"),\n }\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_predictions: DataDefinition([-1, self.prediction_size], [torch.Tensor], \"Batch of predictions, each represented as probability distribution over classes [BATCH_SIZE x PREDICTION_SIZE]\")\n }\n\n def forward(self, data_streams):\n \"\"\"\n Main forward pass of the ``LeNet5`` model.\n\n :param data_streams: DataStreams({'images',**}), where:\n\n - images: [batch_size, num_channels, width, height]\n\n :type data_streams: ``miprometheus.utils.DataStreams``\n\n :return: Predictions [batch_size, num_classes]\n\n \"\"\"\n # Add noise to weights\n #for _, param in self.named_parameters():\n # if param.requires_grad:\n # #print (name, param.data)\n # #noise = -torch.randn(param.data.shape)*0.3\n # noise = 0.3\n # param.data = param.data * (1 + noise)\n # #print (name, param.data)\n\n\n # Unpack DataStreams.\n img = data_streams[self.key_inputs]\n\n # Pass inputs through layers.\n x = self.conv1(img)\n x = torch.nn.functional.relu(x)\n x = self.maxpool1(x)\n x = self.conv2(x)\n x = torch.nn.functional.relu(x)\n x = self.maxpool2(x)\n x = self.conv3(x)\n x = torch.nn.functional.relu(x)\n x = x.view(-1, 120)\n x = self.linear1(x)\n x = torch.nn.functional.relu(x)\n x = self.linear2(x)\n\n # Log softmax.\n predictions = torch.nn.functional.log_softmax(x, dim=1)\n # Add predictions to datadict.\n data_streams.publish({self.key_predictions: predictions})\n" }, { "path": "ptp/components/publishers/__init__.py", "content": "from .global_variable_publisher import GlobalVariablePublisher\nfrom .stream_file_exporter import StreamFileExporter\n\n__all__ = [\n 'GlobalVariablePublisher',\n 'StreamFileExporter',\n ]\n" }, { "path": "ptp/components/publishers/global_variable_publisher.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom ptp.components.component import Component\nfrom ptp.configuration.configuration_error import ConfigurationError\n\nclass GlobalVariablePublisher(Component):\n \"\"\"\n Component responsible for publishing variables set in configuration file as globals.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object. Loads keys and values of variables and adds them to globals.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, GlobalVariablePublisher, config)\n\n\n # Get list of keys of global variables - can be both list of strings or a single string with comma-separated values.\n keys = self.config[\"keys\"]\n if type(keys) is str:\n keys = keys.replace(\" \",\"\").split(\",\")\n # Get list of values - must be a single value or a list.\n values = self.config[\"values\"]\n\n \n if type(values) is list:\n # Make sure that both are lists.\n if type(keys) is not list or len(keys) != len(values):\n raise ConfigurationError(\"Number of parameters indicated by provided 'keys' must be equal to number of provided 'values'\")\n\n # Publish globals one by one.\n for (key, value) in zip(keys, values):\n self.globals[key] = value\n elif keys != '':\n # Publish single global.\n self.globals[keys[0]] = values\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return { }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return { }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Empty method.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object.\n\n \"\"\"\n pass" }, { "path": "ptp/components/publishers/stream_file_exporter.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom os import path\n\nfrom ptp.configuration.config_parsing import get_value_list_from_dictionary\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass StreamFileExporter(Component):\n \"\"\"\n Utility for exporting contents of streams of a given batch to file.\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the object, retrieves names of input streams and creates the output file in experiment directory.\n\n :param name: Name of the component.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, StreamFileExporter, config)\n\n # Get key mappings for indices.\n self.key_indices = self.stream_keys[\"indices\"]\n\n # Load list of streams names (keys).\n self.input_stream_keys = get_value_list_from_dictionary(\"input_streams\", self.config)\n \n # Get separator.\n self.separator = self.config[\"separator\"]\n\n # Create file where we will write the results.\n filename = self.config[\"filename\"]\n abs_filename = path.join(self.app_state.log_dir, filename)\n self.file = open(abs_filename, 'w')\n\n # Export additional line with separator.\n if self.config[\"export_separator_line_to_csv\"]:\n self.file.write(\"sep={}\\n\".format(self.separator))\n\n # Export header - once, when we will process the first batch.\n self.export_header = self.config[\"export_header_to_csv\"]\n \n\n self.logger.info(\"Writing values from {} streams to {}\".format(self.input_stream_keys, abs_filename))\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Exports values from the indicated streams to file.\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing \"indices\" and other streams that will be exported to file.\n \"\"\"\n # Get batch size.\n indices = data_streams[self.key_indices]\n batch_size = len(indices)\n\n # Check present streams.\n absent_streams = []\n present_streams = []\n for stream_key in self.input_stream_keys:\n if stream_key in data_streams.keys():\n present_streams.append(stream_key)\n else:\n absent_streams.append(stream_key)\n\n # Export header - only once.\n if self.export_header:\n header = ''\n for stream_key in self.input_stream_keys:\n if stream_key in present_streams:\n header = header + stream_key + self.separator\n # Remove the last separator.\n header = header[:-1] + '\\n'\n # Write header to file.\n self.file.write(header)\n # Do it only once.\n self.export_header = False\n\n # Export values to file.\n for i in range(batch_size):\n val_str = ''\n for stream_key in self.input_stream_keys:\n if stream_key in present_streams:\n value = data_streams[stream_key][i]\n # Add value changed to string along with separator.\n val_str = val_str + '{}'.format(value) + self.separator\n # Remove the last separator.\n val_str = val_str[:-1] + '\\n'\n # Write it to file.\n self.file.write(val_str)\n\n # Log values and inform about missing streams.\n if len(absent_streams) > 0:\n self.logger.warning(\"Could not export the following (absent) streams: {}\".format(absent_streams))\n" }, { "path": "ptp/components/statistics/__init__.py", "content": "from .accuracy_statistics import AccuracyStatistics\nfrom .batch_size_statistics import BatchSizeStatistics\nfrom .bleu_statistics import BLEUStatistics\nfrom .precision_recall_statistics import PrecisionRecallStatistics\n\n__all__ = [\n 'AccuracyStatistics',\n 'BatchSizeStatistics',\n 'BLEUStatistics',\n 'PrecisionRecallStatistics',\n ]\n" }, { "path": "ptp/components/statistics/accuracy_statistics.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nimport math\nimport numpy as np\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass AccuracyStatistics(Component):\n \"\"\"\n Class collecting statistics: batch size.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, AccuracyStatistics, config)\n\n # Get stream key mappings.\n self.key_targets = self.stream_keys[\"targets\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n self.key_masks = self.stream_keys[\"masks\"]\n\n # Get prediction distributions/indices flag.\n self.use_prediction_distributions = self.config[\"use_prediction_distributions\"]\n\n # Get masking flag.\n self.use_masking = self.config[\"use_masking\"]\n\n # Get statistics key mappings.\n self.key_accuracy = self.statistics_keys[\"accuracy\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n input_defs = {\n self.key_targets: DataDefinition([-1], [torch.Tensor], \"Batch of targets, each being a single index [BATCH_SIZE]\")\n }\n\n if self.use_prediction_distributions:\n input_defs[self.key_predictions] = DataDefinition([-1, -1], [torch.Tensor], \"Batch of predictions, represented as tensor with probability distribution over classes [BATCH_SIZE x NUM_CLASSES]\")\n else: \n input_defs[self.key_predictions] = DataDefinition([-1], [torch.Tensor], \"Batch of predictions, represented as tensor with indices of predicted answers [BATCH_SIZE]\")\n \n if self.use_masking:\n input_defs[self.key_masks] = DataDefinition([-1], [torch.Tensor], \"Batch of masks [BATCH_SIZE]\")\n return input_defs\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {}\n\n\n def __call__(self, data_streams):\n \"\"\"\n Call method - empty for all statistics.\n \"\"\"\n pass\n\n\n def calculate_accuracy(self, data_streams):\n \"\"\"\n Calculates accuracy equal to mean number of correct classification in a given batch.\n\n :param data_streams: DataStreams containing the targets.\n :type data_streams: DataStreams\n\n :return: Accuracy.\n\n \"\"\"\n # Get targets.\n targets = data_streams[self.key_targets].data.cpu().numpy()\n\n if self.use_prediction_distributions:\n # Get indices of the max log-probability.\n preds = data_streams[self.key_predictions].max(1)[1].data.cpu().numpy()\n else: \n preds = data_streams[self.key_predictions].data.cpu().numpy()\n\n # Calculate the correct predictinos.\n correct = np.equal(preds, targets)\n\n #print(\" Target: {}\\n Prediction: {}\\n Correct: {}\\n\".format(targets, preds, correct))\n\n if self.use_masking:\n # Get masks from inputs.\n masks = data_streams[self.key_masks].data.cpu().numpy()\n correct = correct * masks\n batch_size = masks.sum() \n else:\n batch_size = preds.shape[0] \n \n #print(\" Mask: {}\\n Masked Correct: {}\\n\".format(masks, correct))\n\n # Simply sum the correct values.\n num_correct = correct.sum()\n\n #print(\" num_correct: {}\\n batch_size: {}\\n\".format(num_correct, batch_size))\n\n # Normalize by batch size.\n if batch_size > 0:\n accuracy = num_correct / batch_size\n else:\n accuracy = 0\n\n return accuracy, batch_size\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds 'accuracy' statistics to ``StatisticsCollector``.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col.add_statistics(self.key_accuracy, '{:6.4f}')\n stat_col.add_statistics(self.key_accuracy+'_support', None)\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics (accuracy and support set size) for given episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n acc, batch_size = self.calculate_accuracy(data_streams)\n stat_col[self.key_accuracy] = acc\n stat_col[self.key_accuracy+'_support'] = batch_size\n \n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds aggregator summing samples from all collected batches.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n stat_agg.add_aggregator(self.key_accuracy, '{:7.5f}') # represents the average accuracy\n #stat_agg.add_aggregator(self.key_accuracy+'_min', '{:7.5f}')\n #stat_agg.add_aggregator(self.key_accuracy+'_max', '{:7.5f}')\n stat_agg.add_aggregator(self.key_accuracy+'_std', '{:7.5f}')\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates samples from all collected batches.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n accuracies = stat_col[self.key_accuracy]\n supports = stat_col[self.key_accuracy+'_support']\n\n # Special case - no samples!\n if sum(supports) == 0:\n stat_agg[self.key_accuracy] = 0\n stat_agg[self.key_accuracy+'_std'] = 0\n\n else: \n # Calculate weighted precision.\n accuracies_avg = np.average(accuracies, weights=supports)\n accuracies_var = np.average((accuracies-accuracies_avg)**2, weights=supports)\n\n stat_agg[self.key_accuracy] = accuracies_avg\n #stat_agg[self.key_accuracy+'_min'] = np.min(accuracies)\n #stat_agg[self.key_accuracy+'_max'] = np.max(accuracies)\n stat_agg[self.key_accuracy+'_std'] = math.sqrt(accuracies_var)\n" }, { "path": "ptp/components/statistics/batch_size_statistics.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass BatchSizeStatistics(Component):\n \"\"\"\n Class collecting statistics: batch size.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Batch size name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, BatchSizeStatistics, config)\n\n # Set key mappings.\n self.key_indices = self.stream_keys[\"indices\"]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of indices [BATCH_SIZE] x [1]\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {}\n\n\n def __call__(self, data_streams):\n \"\"\"\n Call method - empty for all statistics.\n \"\"\"\n pass\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds 'batch_size' statistics to ``StatisticsCollector``.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col.add_statistics('batch_size', '{:06d}')\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics (batch_size) for given episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col['batch_size'] = len(data_streams[self.key_indices])\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds aggregator summing number of samples from all collected batches.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n stat_agg.add_aggregator('samples_aggregated', '{:006d}')\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates number of samples from all collected batches.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n stat_agg['samples_aggregated'] = sum(stat_col['batch_size'])\n" }, { "path": "ptp/components/statistics/bleu_statistics.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nimport math\nimport numpy as np\nfrom nltk.translate.bleu_score import sentence_bleu\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass BLEUStatistics(Component):\n \"\"\"\n Class collecting statistics: BLEU (Bilingual Evaluation Understudy Score).\n\n It accepts targets and predictions represented as indices of words and uses the provided word mappings to change those into words used finally for calculation of BLEU similarity.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, BLEUStatistics, config)\n\n # Get stream key mappings.\n self.key_targets = self.stream_keys[\"targets\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n self.key_masks = self.stream_keys[\"masks\"]\n\n # Get prediction distributions/indices flag.\n self.use_prediction_distributions = self.config[\"use_prediction_distributions\"]\n\n # Get masking flag.\n #self.use_masking = self.config[\"use_masking\"]\n\n # Get ignored words\n self.ignored_words = self.config[\"ignored_words\"]\n\n # Retrieve word mappings from globals.\n word_to_ix = self.globals[\"word_mappings\"]\n # Construct reverse mapping for faster processing.\n self.ix_to_word = dict((v,k) for k,v in word_to_ix.items())\n\n # Get masking flag.\n self.weights = self.config[\"weights\"]\n\n\n # Get statistics key mappings.\n self.key_bleu = self.statistics_keys[\"bleu\"]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n # Add targets.\n input_defs = {\n self.key_targets: DataDefinition([-1, -1], [torch.Tensor], \"Batch of sentences represented as a single tensor of indices of particular words [BATCH_SIZE x SEQ_LENGTH]\"),\n }\n # Add predictions.\n if self.use_prediction_distributions:\n input_defs[self.key_predictions] = DataDefinition([-1, -1, -1], [torch.Tensor], \"Batch of predictions, represented as tensor with sequences of probability distributions over classes [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES]\")\n else: \n input_defs[self.key_predictions] = DataDefinition([-1, -1], [torch.Tensor], \"Batch of predictions, represented as tensor with sequences of indices of predicted answers [BATCH_SIZE x SEQ_LENGTH]\")\n # Add masks.\n #if self.use_masking:\n # input_defs[self.key_masks] = DataDefinition([-1, -1], [torch.Tensor], \"Batch of masks (separate mask for each sequence in the batch) [BATCH_SIZE x SEQ_LENGTH]\")\n return input_defs\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {}\n\n\n def __call__(self, data_streams):\n \"\"\"\n Call method - empty for all statistics.\n \"\"\"\n pass\n\n\n def calculate_BLEU(self, data_streams):\n \"\"\"\n Calculates BLEU for predictions of a given batch.\n\n :param data_streams: DataStreams containing the targets and predictions (and optionally masks).\n :type data_streams: DataStreams\n\n :return: Accuracy.\n\n \"\"\"\n # Get targets.\n targets = data_streams[self.key_targets].data.cpu().numpy().tolist()\n\n if self.use_prediction_distributions:\n # Get indices of the max log-probability.\n preds = data_streams[self.key_predictions].max(-1)[1].data.cpu().numpy().tolist()\n else: \n preds = data_streams[self.key_predictions].data.cpu().numpy().tolist()\n\n #if self.use_masking:\n # # Get masks from inputs.\n # masks = data_streams[self.key_masks].data.cpu().numpy().tolist()\n #else:\n # batch_size = preds.shape[0] \n \n # Calculate the correct predictinos.\n scores = []\n\n #print(\"targets ({}): {}\\n\".format(len(targets), targets[0]))\n #print(\"preds ({}): {}\\n\".format(len(preds), preds[0]))\n\n for target_indices, pred_indices in zip(targets, preds):\n # Change target indices to words.\n target_words = []\n for t_ind in target_indices:\n if t_ind in self.ix_to_word.keys():\n w = self.ix_to_word[t_ind]\n if w not in self.ignored_words:\n target_words.append(w)\n # Change prediction indices to words.\n pred_words = []\n for p_ind in pred_indices:\n if p_ind in self.ix_to_word.keys():\n w = self.ix_to_word[p_ind]\n if w not in self.ignored_words:\n pred_words.append(w)\n # Calculate BLEU.\n scores.append(sentence_bleu([target_words], pred_words, self.weights))\n #print(\"TARGET: {}\\n\".format(target_words))\n #print(\"PREDICTION: {}\\n\".format(pred_words))\n #print(\"BLEU: {}\\n\".format(scores[-1]))\n\n \n # Get batch size.\n batch_size = len(targets)\n\n # Normalize by batch size.\n if batch_size > 0:\n score = sum(scores) / batch_size\n else:\n score = 0\n\n return score\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds 'accuracy' statistics to ``StatisticsCollector``.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col.add_statistics(self.key_bleu, '{:6.4f}')\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics (batch_size) for given episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n stat_col[self.key_bleu] = self.calculate_BLEU(data_streams)\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds aggregator summing samples from all collected batches.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n stat_agg.add_aggregator(self.key_bleu, '{:7.5f}') # represents the average accuracy\n #stat_agg.add_aggregator(self.key_bleu+'_min', '{:7.5f}')\n #stat_agg.add_aggregator(self.key_bleu+'_max', '{:7.5f}')\n stat_agg.add_aggregator(self.key_bleu+'_std', '{:7.5f}')\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates samples from all collected batches.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n scores = stat_col[self.key_bleu]\n\n # Check if batch size was collected.\n if \"batch_size\" in stat_col.keys():\n batch_sizes = stat_col['batch_size']\n\n # Calculate weighted precision.\n scores_avg = np.average(scores, weights=batch_sizes)\n scores_var = np.average((scores-scores_avg)**2, weights=batch_sizes)\n\n stat_agg[self.key_bleu] = scores_avg\n #stat_agg[self.key_bleu+'_min'] = np.min(scores)\n #stat_agg[self.key_bleu+'_max'] = np.max(scores)\n stat_agg[self.key_bleu+'_std'] = math.sqrt(scores_var)\n else:\n # Else: use simple mean.\n stat_agg[self.key_bleu] = np.mean(scores)\n #stat_agg[self.key_bleu+'_min'] = np.min(scores)\n #stat_agg[self.key_bleu+'_max'] = np.max(scores)\n stat_agg[self.key_bleu+'_std'] = np.std(scores)\n # But inform user about that!\n self.logger.warning(\"Aggregated statistics might contain errors due to the lack of information about sizes of aggregated batches\")\n" }, { "path": "ptp/components/statistics/precision_recall_statistics.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nimport numpy as np\nimport math\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass PrecisionRecallStatistics(Component):\n \"\"\"\n Class collecting statistics: batch size.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, PrecisionRecallStatistics, config)\n\n # Get stream key mappings.\n self.key_targets = self.stream_keys[\"targets\"]\n self.key_predictions = self.stream_keys[\"predictions\"]\n self.key_masks = self.stream_keys[\"masks\"]\n\n # Get prediction distributions/indices flag.\n self.use_prediction_distributions = self.config[\"use_prediction_distributions\"]\n\n # Get masking flag.\n self.use_masking = self.config[\"use_masking\"]\n\n # Get statistics key mappings.\n self.key_precision = self.statistics_keys[\"precision\"]\n self.key_recall = self.statistics_keys[\"recall\"]\n self.key_f1score = self.statistics_keys[\"f1score\"]\n\n \n # Get (or create) vocabulary.\n if self.config[\"use_word_mappings\"]:\n # Get labels from word mappings.\n self.labels = []\n self.index_mappings = {}\n # Assume they are ordered, starting from 0.\n for i,(word,index) in enumerate(self.globals[\"word_mappings\"].items()):\n self.labels.append(word)\n self.index_mappings[index] = i\n # Set number of classes by looking at labels.\n self.num_classes = len(self.labels)\n else:\n # Get the number of possible outputs.\n self.num_classes = self.globals[\"num_classes\"]\n self.labels = list(range(self.num_classes))\n self.index_mappings = {i: i for i in range(self.num_classes)}\n\n # Check display options.\n self.show_confusion_matrix = self.config[\"show_confusion_matrix\"]\n self.show_class_scores = self.config[\"show_class_scores\"]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n input_defs = {\n self.key_targets: DataDefinition([-1], [torch.Tensor], \"Batch of targets, each being a single index [BATCH_SIZE]\")\n }\n\n if self.use_prediction_distributions:\n input_defs[self.key_predictions] = DataDefinition([-1, -1], [torch.Tensor], \"Batch of predictions, represented as tensor with probability distribution over classes [BATCH_SIZE x NUM_CLASSES]\")\n else: \n input_defs[self.key_predictions] = DataDefinition([-1], [torch.Tensor], \"Batch of predictions, represented as tensor with indices of predicted answers [BATCH_SIZE]\")\n\n if self.use_masking:\n input_defs[self.key_masks] = DataDefinition([-1], [torch.Tensor], \"Batch of masks [BATCH_SIZE]\")\n return input_defs\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {}\n\n def __call__(self, data_streams):\n \"\"\"\n Calculates precission recall statistics.\n\n :param data_streams: DataStreams containing the targets.\n :type data_streams: DataStreams\n\n \"\"\"\n # Use worker interval.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n\n # Calculate all four statistics.\n confusion_matrix, precisions, recalls, f1scores, supports = self.calculate_statistics(data_streams)\n\n if self.show_confusion_matrix:\n self.logger.info(\"Confusion matrix:\\n{}\".format(confusion_matrix))\n\n # Calculate weighted averages.\n support_sum = sum(supports)\n if support_sum > 0:\n precision_avg = sum([pi*si for (pi,si) in zip(precisions,supports)]) / support_sum \n recall_avg = sum([ri*si for (ri,si) in zip(recalls,supports)]) / support_sum\n f1score_avg = sum([fi*si for (fi,si) in zip(f1scores,supports)]) / support_sum\n else:\n precision_avg = 0\n recall_avg = 0\n f1score_avg = 0\n\n # Log class scores.\n if self.show_class_scores:\n log_str = \"\\n| Precision | Recall | F1Score | Support | Label\\n\"\n log_str+= \"|-----------|--------|---------|---------|-------\\n\"\n for i in range(self.num_classes):\n log_str += \"| {:05.4f} | {:05.4f} | {:05.4f} | {:5d} | {}\\n\".format(\n precisions[i], recalls[i], f1scores[i], supports[i], self.labels[i])\n log_str+= \"|-----------|--------|---------|---------|-------\\n\"\n log_str += \"| {:05.4f} | {:05.4f} | {:05.4f} | {:5d} | Weighted Avg\\n\".format(\n precision_avg, recall_avg, f1score_avg, support_sum)\n self.logger.info(log_str)\n\n\n def calculate_statistics(self, data_streams):\n \"\"\"\n Calculates confusion_matrix, precission, recall, f1score and support statistics.\n\n :param data_streams: DataStreams containing the targets.\n :type data_streams: DataStreams\n\n :return: Calculated statistics.\n \"\"\"\n targets = data_streams[self.key_targets].data.cpu().numpy()\n #print(\"Targets :\", targets)\n\n if self.use_prediction_distributions:\n # Get indices of the max log-probability.\n preds = data_streams[self.key_predictions].max(1)[1].data.cpu().numpy()\n else: \n preds = data_streams[self.key_predictions].data.cpu().numpy()\n #print(\"Predictions :\", preds)\n\n if self.use_masking:\n # Get masks from inputs.\n masks = data_streams[self.key_masks].data.cpu().numpy()\n else:\n # Create vector full of ones.\n masks = np.ones(targets.shape[0])\n\n # Create the confusion matrix, use SciKit learn order:\n # Column - predicted class\n #print(self.index_mappings)\n # Row - target (actual) class\n confusion_matrix = np.zeros([self.num_classes, self.num_classes], dtype=int)\n for i, (target, pred) in enumerate(zip(targets, preds)):\n #print(\"T: \",target)\n #print(\"P: \",pred)\n # If both indices are ok.\n if target in self.index_mappings.keys() and pred in self.index_mappings.keys():\n #print(self.index_mappings[target])\n #print(self.index_mappings[pred])\n confusion_matrix[self.index_mappings[target]][self.index_mappings[pred]] += 1 * masks[i]\n\n # Calculate true positive (TP), eqv. with hit.\n tp = np.zeros([self.num_classes], dtype=int)\n for i in range(self.num_classes):\n tp[i] = confusion_matrix[i][i]\n #print(\"TP = \",tp) \n\n # Calculate false positive (FP) eqv. with false alarm, Type I error\n # Predictions that incorrectly labelled as belonging to a given class.\n # Sum wrong predictions along the column.\n fp = np.sum(confusion_matrix, axis=0) - tp\n #print(\"FP = \",fp)\n\n # Calculate false negative (FN), eqv. with miss, Type II error\n # The target belonged to a given class, but it wasn't correctly labeled.\n # Sum wrong predictions along the row.\n fn = np.sum(confusion_matrix, axis=1) - tp\n #print(\"FN = \",fn) \n\n # Precision is the fraction of events where we correctly declared i\n # out of all instances where the algorithm declared i.\n precisions = [float(tpi) / float(tpi+fpi) if (tpi+fpi) > 0 else 0.0 for (tpi,fpi) in zip(tp,fp)]\n\n # Recall is the fraction of events where we correctly declared i \n # out of all of the cases where the true of state of the world is i.\n recalls = [float(tpi) / float(tpi+fni) if (tpi+fni) > 0 else 0.0 for (tpi,fni) in zip(tp,fn)]\n\n # Calcualte f1-scores.\n f1scores = [ 2 * pi * ri / float(pi+ri) if (pi+ri) > 0 else 0.0 for (pi,ri) in zip(precisions,recalls)]\n\n # Get support.\n supports = np.sum(confusion_matrix, axis=1)\n\n #print('precision: {}'.format(precision))\n #print('recall: {}'.format(recall))\n #print('f1score: {}'.format(f1score))\n #print('support: {}'.format(support))\n\n return confusion_matrix, precisions, recalls, f1scores, supports\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds 'accuracy' statistics to ``StatisticsCollector``.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n # Those will be displayed.\n stat_col.add_statistics(self.key_precision, '{:05.4f}')\n stat_col.add_statistics(self.key_recall, '{:05.4f}')\n stat_col.add_statistics(self.key_f1score, '{:05.4f}')\n # That one will be collected and used by aggregator.\n stat_col.add_statistics(self.key_f1score+'_support', None)\n\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics (batch_size) for given episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n # Calculate all four statistics.\n _, precisions, recalls, f1scores, supports = self.calculate_statistics(data_streams)\n\n # Calculate weighted averages.\n precision_sum = sum([pi*si for (pi,si) in zip(precisions,supports)])\n recall_sum = sum([ri*si for (ri,si) in zip(recalls,supports)])\n f1score_sum = sum([fi*si for (fi,si) in zip(f1scores,supports)])\n support_sum = sum(supports)\n\n if support_sum > 0:\n precision_avg = precision_sum / support_sum \n recall_avg = recall_sum / support_sum\n f1score_avg = f1score_sum / support_sum\n else:\n precision_avg = 0\n recall_avg = 0\n f1score_avg = 0\n\n # Export averages to statistics.\n stat_col[self.key_precision] = precision_avg\n stat_col[self.key_recall] = recall_avg\n stat_col[self.key_f1score] = f1score_avg\n\n # Export support to statistics.\n stat_col[self.key_f1score+'_support'] = support_sum\n\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds aggregator summing samples from all collected batches.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n stat_agg.add_aggregator(self.key_precision, '{:05.4f}') \n stat_agg.add_aggregator(self.key_precision+'_std', '{:05.4f}')\n stat_agg.add_aggregator(self.key_recall, '{:05.4f}') \n stat_agg.add_aggregator(self.key_recall+'_std', '{:05.4f}')\n stat_agg.add_aggregator(self.key_f1score, '{:05.4f}') \n stat_agg.add_aggregator(self.key_f1score+'_std', '{:05.4f}')\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates samples from all collected batches.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n\n \"\"\"\n precision_sums = stat_col[self.key_precision]\n recall_sums = stat_col[self.key_recall]\n f1score_sums = stat_col[self.key_f1score]\n supports = stat_col[self.key_f1score+'_support']\n\n # Special case - no samples!\n if sum(supports) == 0:\n stat_agg[self.key_precision] = 0\n stat_agg[self.key_precision+'_std'] = 0\n stat_agg[self.key_recall] = 0\n stat_agg[self.key_recall+'_std'] = 0\n stat_agg[self.key_f1score] = 0\n stat_agg[self.key_f1score+'_std'] = 0\n\n else: \n # Else: calculate weighted precision.\n precisions_avg = np.average(precision_sums, weights=supports)\n precisions_var = np.average((precision_sums-precisions_avg)**2, weights=supports)\n \n stat_agg[self.key_precision] = precisions_avg\n stat_agg[self.key_precision+'_std'] = math.sqrt(precisions_var)\n\n # Calculate weighted recall.\n recalls_avg = np.average(recall_sums, weights=supports)\n recalls_var = np.average((recall_sums-recalls_avg)**2, weights=supports)\n\n stat_agg[self.key_recall] = recalls_avg\n stat_agg[self.key_recall+'_std'] = math.sqrt(recalls_var)\n\n # Calculate weighted f1 score.\n f1scores_avg = np.average(f1score_sums, weights=supports)\n f1scores_var = np.average((f1score_sums-f1scores_avg)**2, weights=supports)\n\n stat_agg[self.key_f1score] = f1scores_avg\n stat_agg[self.key_f1score+'_std'] = math.sqrt(f1scores_var)\n" }, { "path": "ptp/components/tasks/image_text_to_class/__init__.py", "content": "from .clevr import CLEVR\nfrom .gqa import GQA\nfrom .vqa_med_2019 import VQAMED2019\n\n__all__ = [\n 'CLEVR',\n 'GQA',\n 'VQAMED2019',\n ]\n" }, { "path": "ptp/components/tasks/image_text_to_class/clevr.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport json\nfrom PIL import Image\n\nimport torch\nfrom torchvision import transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\n#from ptp.components.mixins.io import save_nparray_to_csv_file\nfrom ptp.configuration.config_parsing import get_value_from_dictionary, get_value_list_from_dictionary\nfrom ptp.configuration.configuration_error import ConfigurationError\n\nclass CLEVR(Task):\n \"\"\"\n Task providing data associated with CLEVR (Compositional Language andElementary Visual Reasoning) diagnostics dataset\n\n The dataset consists of three splits:\n - A training set of 70,000 images and 699,989 questions\n - A validation set of 15,000 images and 149,991 questions\n - A test set of 15,000 images and 14,988 questions\n - Answers for all train and val questions\n - Scene graph annotations for train and val images giving ground-truth locations, attributes, and relationships for objects\n - Functional program representations for all training and validation images\n\n CLEVR contains a total of 90 question families, eachwith a single program template and an average of four texttemplates.\n Those are further aggregated into 13 Question Types:\n - Querying attributes (Size, Color, Material, Shape)\n - Comparing attributes (Size, Color, Material, Shape)\n - Existence\n - Counting\n - Integer comparison (Equal, Less, More)\n\n For more details please refer to the associated _website or _paper for more details.\n\n .. _website: https://cs.stanford.edu/people/jcjohns/clevr/\n\n .._paper: https://arxiv.org/pdf/1612.06890\n\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes task object. Calls base constructor. Downloads the dataset if not present and loads the adequate files depending on the mode.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n Task.__init__(self, name, CLEVR, config)\n\n # Get key mappings of all output streams.\n self.key_images = self.stream_keys[\"images\"]\n self.key_image_ids = self.stream_keys[\"image_ids\"]\n self.key_questions = self.stream_keys[\"questions\"]\n self.key_answers = self.stream_keys[\"answers\"]\n self.key_question_type_ids = self.stream_keys[\"question_type_ids\"]\n self.key_question_type_names = self.stream_keys[\"question_type_names\"]\n\n # Get flag informing whether we want to stream images or not.\n self.stream_images = self.config['stream_images']\n\n # Check the resize image option.\n if \"resize_image\" in self.config:\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n self.depth = 3\n resize = True\n else:\n # Use original image dimensions.\n self.height = 480 \n self.width = 320 \n self.depth = 3\n resize = False\n self.logger.info(\"Setting image size to [D x H x W]: {} x {} x {}\".format(self.depth, self.height, self.width))\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_depth\"] = self.depth\n\n # Get image preprocessing.\n self.image_preprocessing = get_value_list_from_dictionary(\n \"image_preprocessing\", self.config,\n 'none | normalize | all'.split(\" | \")\n )\n if 'none' in self.image_preprocessing:\n self.image_preprocessing = []\n if 'all' in self.image_preprocessing:\n self.image_preprocessing = ['normalize']\n\n if resize:\n # Add resize as transformation.\n self.image_preprocessing = [\"resize\"] + self.image_preprocessing\n self.logger.info(\"Applied image preprocessing: {}\".format(self.image_preprocessing))\n\n # Mapping of question subtypes to types (not used, but keeping it just in case).\n #self.question_subtype_to_type_mapping = {\n # 'query_size': 'query_attribute',\n # 'equal_size': 'compare_attribute',\n # 'query_shape': 'query_attribute',\n # 'query_color': 'query_attribute',\n # 'greater_than': 'compare_integer',\n # 'equal_material': 'compare_attribute',\n # 'equal_color': 'compare_attribute',\n # 'equal_shape': 'compare_attribute',\n # 'less_than': 'compare_integer',\n # 'count': 'count',\n # 'exist': 'exist',\n # 'equal_integer': 'compare_integer',\n # 'query_material': 'query_attribute'}\n\n # Mapping of question subtypes to types.\n self.question_subtype_to_id_mapping = {\n 'query_size': 0,\n 'equal_size': 1,\n 'query_shape': 2,\n 'query_color': 3,\n 'greater_than': 4,\n 'equal_material': 5,\n 'equal_color': 6,\n 'equal_shape': 7,\n 'less_than': 8,\n 'count': 9,\n 'exist': 10,\n 'equal_integer': 11,\n 'query_material': 12}\n\n # Mapping of question families to subtypes.\n self.question_family_id_to_subtype_mapping = {\n 0: \"equal_integer\", 1: \"less_than\", 2: \"greater_than\", 3: \"equal_integer\", 4: \"less_than\", 5: \"greater_than\", 6: \"equal_integer\", 7: \"less_than\", 8: \"greater_than\", 9: \"equal_size\",\n 10: \"equal_color\", 11: \"equal_material\", 12: \"equal_shape\", 13: \"equal_size\", 14: \"equal_size\", 15: \"equal_size\", 16: \"equal_color\", 17: \"equal_color\", 18: \"equal_color\", 19: \"equal_material\",\n 20: \"equal_material\", 21: \"equal_material\", 22: \"equal_shape\", 23: \"equal_shape\", 24: \"equal_shape\", 25: \"count\", 26: \"exist\", 27: \"query_size\", 28: \"query_shape\", 29: \"query_color\",\n 30: \"query_material\", 31: \"count\", 32: \"query_size\", 33: \"query_color\", 34: \"query_material\", 35: \"query_shape\", 36: \"exist\", 37: \"exist\", 38: \"exist\", 39: \"exist\",\n 40: \"count\", 41: \"count\", 42: \"count\", 43: \"count\", 44: \"exist\", 45: \"exist\", 46: \"exist\", 47: \"exist\", 48: \"count\", 49: \"count\",\n 50: \"count\", 51: \"count\", 52: \"query_color\", 53: \"query_material\", 54: \"query_shape\", 55: \"query_size\", 56: \"query_material\", 57: \"query_shape\", 58: \"query_size\", 59: \"query_color\",\n 60: \"query_shape\", 61: \"query_size\", 62: \"query_color\", 63: \"query_material\", 64: \"count\", 65: \"count\", 66: \"count\", 67: \"count\", 68: \"count\", 69: \"count\",\n 70: \"count\", 71: \"count\", 72: \"count\", 73: \"exist\", 74: \"query_size\", 75: \"query_color\", 76: \"query_material\", 77: \"query_shape\", 78: \"count\", 79: \"exist\",\n 80: \"query_size\", 81: \"query_color\", 82: \"query_material\", 83: \"query_shape\", 84: \"count\", 85: \"exist\", 86: \"query_shape\", 87: \"query_material\", 88: \"query_color\", 89: \"query_size\"}\n\n # Finally, \"merge\" those two.\n self.question_family_id_to_subtype_id_mapping = { key: self.question_subtype_to_id_mapping[value] for key, value in self.question_family_id_to_subtype_mapping.items() }\n\n\n # Get the absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get split.\n split = get_value_from_dictionary('split', self.config, \"training | validation | test | cogent_a_training | cogent_a_validation | cogent_b_validation\".split(\" | \"))\n\n # Set split-dependent data.\n if split == 'training':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, \"questions\", 'CLEVR_train_questions.json')\n self.split_image_folder = os.path.join(self.data_folder, \"images\", \"train\")\n\n elif split == 'validation':\n # Validation split folder and file with data question.\n data_file = os.path.join(self.data_folder, \"questions\", 'CLEVR_val_questions.json')\n self.split_image_folder = os.path.join(self.data_folder, \"images\", \"val\")\n\n elif split == 'test':\n # Test split folder and file with data question.\n data_file = os.path.join(self.data_folder, \"questions\", 'CLEVR_test_questions.json')\n self.split_image_folder = os.path.join(self.data_folder, \"images\", \"test\")\n\n else: # cogent\n raise ConfigurationError(\"Split {} not supported yet\".format(split))\n\n # Load dataset.\n self.dataset = self.load_dataset(data_file)\n \n # Display exemplary sample.\n i = 0\n sample = self.dataset[i]\n # Check if this is a test set.\n if \"answer\" not in sample.keys():\n sample[\"answer\"] = \"\"\n sample[self.key_question_type_ids] = -1\n sample[self.key_question_type_names] = \"\"\n else:\n sample[self.key_question_type_ids] = self.question_family_id_to_subtype_id_mapping[sample[\"question_family_index\"]]\n sample[self.key_question_type_names] = self.question_family_id_to_subtype_mapping[sample[\"question_family_index\"]]\n\n self.logger.info(\"Exemplary sample {} ({}):\\n question_type: {} ({})\\n image_ids: {}\\n question: {}\\n answer: {}\".format(\n i, sample[\"question_index\"],\n sample[self.key_question_type_ids],\n sample[self.key_question_type_names],\n sample[\"image_filename\"],\n sample[\"question\"],\n sample[\"answer\"]\n ))\n\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n # Add all \"standard\" streams.\n d = {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_image_ids: DataDefinition([-1, 1], [list, str], \"Batch of image names, each being a single word [BATCH_SIZE] x [STRING]\"),\n self.key_question_type_ids: DataDefinition([-1], [torch.Tensor], \"Batch of target question type indices, each being a single index [BATCH_SIZE]\"),\n self.key_question_type_names: DataDefinition([-1, 1], [list, str], \"Batch of target question type names, each being a single word [BATCH_SIZE] x [STRING]\"),\n }\n \n # Return images only when required.\n if self.stream_images:\n d[self.key_images] = DataDefinition([-1, self.depth, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\")\n\n # Add stream with questions.\n d[self.key_questions] = DataDefinition([-1, 1], [list, str], \"Batch of questions, each being a string consisting of many words [BATCH_SIZE] x [STRING]\")\n\n # Add stream with answers.\n d[self.key_answers]= DataDefinition([-1, 1], [list, str], \"Batch of target answers, each being a string consisting of sinlge word (label) [BATCH_SIZE] x [STRING]\")\n\n return d\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n\n def load_dataset(self, source_data_file):\n \"\"\"\n Loads the dataset from source file\n\n :param source_data_file: jSON file with image ids, questions, answers, scene graphs, etc.\n\n \"\"\"\n self.logger.info(\"Loading dataset from:\\n {}\".format(source_data_file))\n dataset = []\n\n with open(source_data_file) as f:\n self.logger.info(\"Loading samples from '{}'...\".format(source_data_file))\n dataset = json.load(f)['questions']\n\n self.logger.info(\"Loaded dataset consisting of {} samples\".format(len(dataset)))\n return dataset\n\n\n def get_image(self, img_id):\n \"\"\"\n Function loads and returns image along with its size.\n Additionally, it performs all the required transformations.\n\n :param img_id: Identifier of the images.\n :param img_folder: Path to the image.\n\n :return: image (Tensor)\n \"\"\"\n\n # Load the image.\n img = Image.open(os.path.join(self.split_image_folder, img_id)).convert('RGB')\n\n image_transformations_list = []\n\n # Optional: resize.\n if 'resize' in self.image_preprocessing:\n image_transformations_list.append(transforms.Resize([self.height,self.width]))\n\n # Add obligatory transformation.\n image_transformations_list.append(transforms.ToTensor())\n\n # Optional: normalization.\n if 'normalize' in self.image_preprocessing:\n # Use normalization that the pretrained models from TorchVision require.\n image_transformations_list.append(transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))\n\n # Resize the image and transform to Torch Tensor.\n transforms_com = transforms.Compose(image_transformations_list)\n # Apply transformations.\n img = transforms_com(img)\n\n # Return image.\n return img\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a single sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: DataStreams({'indices', 'images', 'images_ids','questions', 'answers', 'question_type_ids', 'question_type_names'})\n \"\"\"\n # Get item.\n item = self.dataset[index]\n\n # Create the resulting sample (data dict).\n data_streams = self.create_data_streams(index)\n\n # Load and stream the image ids.\n img_id = item[\"image_filename\"]\n data_streams[self.key_image_ids] = img_id\n\n # Load the adequate image - only when required.\n if self.stream_images:\n img = self.get_image(img_id)\n # Image related variables.\n data_streams[self.key_images] = img\n\n # Return question.\n data_streams[self.key_questions] = item[\"question\"]\n\n # Return answer. \n if \"answer\" in item.keys():\n data_streams[self.key_answers] = item[\"answer\"]\n else:\n data_streams[self.key_answers] = \"\"\n\n # Question type related variables.\n if \"question_family_index\" in item.keys():\n data_streams[self.key_question_type_ids] = self.question_family_id_to_subtype_id_mapping[item[\"question_family_index\"]]\n data_streams[self.key_question_type_names] = self.question_family_id_to_subtype_mapping[item[\"question_family_index\"]]\n else:\n data_streams[self.key_question_type_ids] = -1\n data_streams[self.key_question_type_names] = \"\"\n\n # Return sample.\n return data_streams\n\n\n def collate_fn(self, batch):\n \"\"\"\n Combines a list of DataStreams (retrieved with :py:func:`__getitem__`) into a batch.\n\n :param batch: list of individual samples to combine\n :type batch: list\n\n :return: DataStreams({'indices', 'images', 'images_ids','questions', 'answers', 'category_ids', 'image_sizes'})\n\n \"\"\"\n # Collate indices.\n data_streams = self.create_data_streams([sample[self.key_indices] for sample in batch])\n\n # Stack images.\n data_streams[self.key_image_ids] = [item[self.key_image_ids] for item in batch]\n if self.stream_images:\n data_streams[self.key_images] = torch.stack([item[self.key_images] for item in batch]).type(torch.FloatTensor)\n\n # Collate lists/lists of lists.\n data_streams[self.key_questions] = [item[self.key_questions] for item in batch]\n data_streams[self.key_answers] = [item[self.key_answers] for item in batch]\n\n # Stack categories.\n data_streams[self.key_question_type_ids] = torch.tensor([item[self.key_question_type_ids] for item in batch])\n data_streams[self.key_question_type_names] = [item[self.key_question_type_names] for item in batch]\n\n # Return collated dict.\n return data_streams\n" }, { "path": "ptp/components/tasks/image_text_to_class/gqa.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport json\nimport tqdm\nfrom PIL import Image\n\nimport torch\nfrom torchvision import transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\nfrom ptp.configuration.config_parsing import get_value_from_dictionary, get_value_list_from_dictionary\nfrom ptp.configuration.configuration_error import ConfigurationError\n\nclass GQA(Task):\n \"\"\"\n Task providing data associated with the GQA dataset (Question Answering on Image Scene Graphs).\n\n The dataset consists of 22M questions about various day-to-day images. \n Each image is associated with a scene graph of the image's objects, attributes and relations.\n Each question is associated with a structured representation of its semantics, a functional program\n that specifies the reasoning steps have to be taken to answer it.\n\n For more details please refer to the associated _website or _paper for more details.\n Test set with answers can be downloaded from a separate repository _repo.\n\n .. _website: https://cs.stanford.edu/people/dorarad/gqa/index.html\n\n .._paper: http://openaccess.thecvf.com/content_CVPR_2019/html/Hudson_GQA_A_New_Dataset_for_Real-World_Visual_Reasoning_and_Compositional_CVPR_2019_paper.html\n\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes task object. Calls base constructor. Downloads the dataset if not present and loads the adequate files depending on the mode.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n Task.__init__(self, name, GQA, config)\n\n # Get key mappings of all output streams.\n self.key_sample_ids = self.stream_keys[\"sample_ids\"]\n self.key_images = self.stream_keys[\"images\"]\n self.key_image_ids = self.stream_keys[\"image_ids\"]\n self.key_questions = self.stream_keys[\"questions\"]\n self.key_answers = self.stream_keys[\"answers\"]\n self.key_full_answers = self.stream_keys[\"full_answers\"]\n\n # Get flag informing whether we want to stream images or not.\n self.stream_images = self.config['stream_images']\n\n # Check the resize image option.\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n self.depth = 3\n self.logger.info(\"Setting image size to [D x H x W]: {} x {} x {}\".format(self.depth, self.height, self.width))\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_depth\"] = self.depth\n\n # Get image preprocessing.\n self.image_preprocessing = get_value_list_from_dictionary(\n \"image_preprocessing\", self.config,\n 'none | normalize | all'.split(\" | \")\n )\n if 'none' in self.image_preprocessing:\n self.image_preprocessing = []\n if 'all' in self.image_preprocessing:\n self.image_preprocessing = ['normalize']\n # Add resize as transformation.\n self.image_preprocessing = [\"resize\"] + self.image_preprocessing\n\n self.logger.info(\"Applied image preprocessing: {}\".format(self.image_preprocessing))\n\n # Get the absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get split.\n split = get_value_from_dictionary('split', self.config, \"training_0 | training | validation | test_dev | test\".split(\" | \"))\n self.split_image_folder = os.path.join(self.data_folder, \"images\")\n\n # Set split-dependent data.\n if split == 'training':\n # Training split folder and file with data question.\n data_files = []\n for i in range(10):\n data_files.append(os.path.join(self.data_folder, \"questions1.2\", \"train_all_questions\", \"train_all_questions_{}.json\".format(i)))\n\n elif split == 'training_0':\n # Validation split folder and file with data question.\n data_files = [ os.path.join(self.data_folder, \"questions1.2\", \"train_all_questions\", \"train_all_questions_0.json\") ]\n self.logger.warning(\"Please remember that this split constitutes only 10 percent of the whole training set!\")\n\n elif split == 'validation':\n # Validation split folder and file with data question.\n data_files = [ os.path.join(self.data_folder, \"questions1.2\", \"val_all_questions.json\") ]\n self.logger.warning(\"Please use 'test_dev' split for validation!\")\n\n elif split == 'test_dev':\n # Validation split folder and file with data question.\n data_files = [ os.path.join(self.data_folder, \"questions1.2\", \"testdev_all_questions.json\") ]\n\n elif split == 'test':\n # Test split folder and file with data question.\n data_files = [ os.path.join(self.data_folder, \"questions1.2\", \"test_all_questions.json\") ]\n\n else:\n raise ConfigurationError(\"Split {} not supported yet\".format(split))\n\n # Load dataset.\n self.dataset = self.load_dataset(data_files)\n \n # Display exemplary sample.\n i = 0\n sample = self.dataset[i]\n # Check if this is a test set.\n self.logger.info(\"Exemplary sample {} ({}):\\n image_ids: {}\\n question: {}\\n answer: {} ({})\".format(\n i,\n sample[self.key_sample_ids],\n sample[self.key_image_ids],\n sample[self.key_questions],\n sample[self.key_answers],\n sample[self.key_full_answers]\n ))\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n # Add all \"standard\" streams.\n d = {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_sample_ids: DataDefinition([-1, 1], [list, int], \"Batch of sample ids [BATCH_SIZE] x [1]\"),\n self.key_image_ids: DataDefinition([-1, 1], [list, str], \"Batch of image names, each being a single word [BATCH_SIZE] x [STRING]\"),\n }\n \n # Return images only when required.\n if self.stream_images:\n d[self.key_images] = DataDefinition([-1, self.depth, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\")\n\n # Add stream with questions.\n d[self.key_questions] = DataDefinition([-1, 1], [list, str], \"Batch of questions, each being a string consisting of many words [BATCH_SIZE] x [STRING]\")\n\n # Add stream with answers.\n d[self.key_answers]= DataDefinition([-1, 1], [list, str], \"Batch of target answers, each being a string consisting of few words (still treated as a single label) [BATCH_SIZE] x [STRING]\")\n d[self.key_full_answers]= DataDefinition([-1, 1], [list, str], \"Batch of target full (long) answers, each being a string consisting of many words [BATCH_SIZE] x [STRING]\")\n\n return d\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n\n def load_dataset(self, source_files):\n \"\"\"\n Loads the dataset from source files.\n\n :param source_files: list of jSON file with image ids, questions, answers, scene graphs, etc.\n\n \"\"\"\n self.logger.info(\"Loading dataset from:\\n {}\".format(source_files))\n dataset = []\n\n # Load and process files, one by one.\n for source_file in source_files:\n with open(source_file) as f:\n self.logger.info(\"Loading samples from '{}'...\".format(source_file))\n json_dataset = json.load(f)\n # Process samples.\n\n # Add tdqm bar.\n t = tqdm.tqdm(total=len(json_dataset))\n for key,value in json_dataset.items():\n # New sample.\n sample = {}\n sample[self.key_sample_ids] = key\n sample[self.key_image_ids] = value[\"imageId\"]\n sample[self.key_questions] = value[\"question\"]\n\n # Return answer. \n if \"answer\" in value.keys():\n sample[self.key_answers] = value[\"answer\"]\n sample[self.key_full_answers] = value[\"fullAnswer\"]\n else:\n # Test set.\n sample[self.key_answers] = \"\"\n sample[self.key_full_answers] = \"\"\n\n # Add to dataset.\n dataset.append(sample)\n t.update()\n # Close the bar.\n t.close()\n\n self.logger.info(\"Loaded dataset consisting of {} samples\".format(len(dataset)))\n return dataset\n\n\n def get_image(self, img_id):\n \"\"\"\n Function loads and returns image along with its size.\n Additionally, it performs all the required transformations.\n\n :param img_id: Identifier of the images.\n :param img_folder: Path to the image.\n\n :return: image (Tensor)\n \"\"\"\n\n # Load the image.\n img = Image.open(os.path.join(self.split_image_folder, img_id+\".jpg\")).convert('RGB')\n\n image_transformations_list = []\n\n # Optional: resize.\n if 'resize' in self.image_preprocessing:\n image_transformations_list.append(transforms.Resize([self.height,self.width]))\n\n # Add obligatory transformation.\n image_transformations_list.append(transforms.ToTensor())\n\n # Optional: normalization.\n if 'normalize' in self.image_preprocessing:\n # Use normalization that the pretrained models from TorchVision require.\n image_transformations_list.append(transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))\n\n # Resize the image and transform to Torch Tensor.\n transforms_com = transforms.Compose(image_transformations_list)\n # Apply transformations.\n img = transforms_com(img)\n\n # Return image.\n return img\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a single sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: DataStreams({'indices', 'sample_ids', images', 'images_ids','questions', 'answers', 'full_answers'})\n \"\"\"\n # Get item.\n item = self.dataset[index]\n\n # Create the resulting sample (data dict).\n data_streams = self.create_data_streams(index)\n\n # Return sample id.\n data_streams[self.key_sample_ids] = item[self.key_sample_ids]\n\n # Load and stream the image ids.\n img_id = item[self.key_image_ids]\n data_streams[self.key_image_ids] = img_id\n\n # Load the adequate image - only when required.\n if self.stream_images:\n data_streams[self.key_images] = self.get_image(img_id)\n\n # Return question.\n data_streams[self.key_questions] = item[self.key_questions]\n\n # Return answers.\n data_streams[self.key_answers] = item[self.key_answers]\n data_streams[self.key_full_answers] = item[self.key_full_answers]\n\n # Return sample.\n return data_streams\n\n\n def collate_fn(self, batch):\n \"\"\"\n Combines a list of DataStreams (retrieved with :py:func:`__getitem__`) into a batch.\n\n :param batch: list of individual samples to combine\n :type batch: list\n\n :return: DataStreams({'indices', 'images', 'images_ids','questions', 'answers', 'category_ids', 'image_sizes'})\n\n \"\"\"\n # Collate indices.\n data_streams = self.create_data_streams([sample[self.key_indices] for sample in batch])\n\n # Collate sample ids.\n data_streams[self.key_sample_ids] = [item[self.key_sample_ids] for item in batch]\n\n # Stack images.\n data_streams[self.key_image_ids] = [item[self.key_image_ids] for item in batch]\n if self.stream_images:\n data_streams[self.key_images] = torch.stack([item[self.key_images] for item in batch]).type(torch.FloatTensor)\n\n # Collate lists/lists of lists.\n data_streams[self.key_questions] = [item[self.key_questions] for item in batch]\n data_streams[self.key_answers] = [item[self.key_answers] for item in batch]\n data_streams[self.key_full_answers] = [item[self.key_full_answers] for item in batch]\n\n # Return collated dict.\n return data_streams\n" }, { "path": "ptp/components/tasks/image_text_to_class/vqa_med_2019.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Chaitanya Shivade, Tomasz Kornuta\"\n\nimport os\nimport string\nimport tqdm\n\nimport pandas as pd\nfrom PIL import Image\nimport numpy as np\nimport nltk\n\nimport torch\nfrom torchvision import transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\nfrom ptp.components.mixins.io import save_nparray_to_csv_file\nfrom ptp.configuration.config_parsing import get_value_list_from_dictionary, get_value_from_dictionary\n\n\nclass VQAMED2019(Task):\n \"\"\"\n Task providing data associated with ImageCLEF VQA 2019 challenge.\n\n The dataset consists of four splits:\n\n - a training set of 3,200 medical images with 12,792 Question-Answer (QA) pairs,\n - a validation set of 500 medical images with 2,000 QA pairs, and\n - a test set of 500 medical images with 500 questions.\n\n Aside of that, there are 4 categories of questions based on:\n - Modality (C1),\n - Plane (C2),\n - Organ System (C3), and\n - Abnormality (C4).\n\n Please see the readme file of the crowdAI dataset section for more detailed information.\n For more details please refer to the associated _website or _crowdai websites for more details.\n Test set with answers can be downloaded from a separate repository _repo.\n\n .. _crowdai: https://www.crowdai.org/challenges/imageclef-2019-vqa-med\n\n .. _website: https://www.imageclef.org/2019/medical/vqa/\n\n .._repo: https://github.com/abachaa/VQA-Med-2019\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes task object. Calls base constructor. Downloads the dataset if not present and loads the adequate files depending on the mode.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n Task.__init__(self, name, VQAMED2019, config)\n\n # (Eventually) download required packages.\n nltk.download('punkt')\n nltk.download('stopwords')\n\n # Get key mappings of all output streams.\n self.key_images = self.stream_keys[\"images\"]\n self.key_image_ids = self.stream_keys[\"image_ids\"]\n self.key_questions = self.stream_keys[\"questions\"]\n self.key_answers = self.stream_keys[\"answers\"]\n self.key_category_ids = self.stream_keys[\"category_ids\"]\n self.key_category_names = self.stream_keys[\"category_names\"]\n self.key_image_sizes = self.stream_keys[\"image_sizes\"]\n\n # Get flag informing whether we want to stream images or not.\n self.stream_images = self.config['stream_images']\n\n # Get flag indicating whether we want to (pre)aload all images at the start.\n self.preload_images = self.config['preload_images']\n\n # Check the desired image size.\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n self.depth = 3\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_depth\"] = self.depth\n\n # Those values will be used to rescale the image_sizes to range (0, 1).\n self.scale_image_height = self.config['scale_image_size'][0]\n self.scale_image_width = self.config['scale_image_size'][1]\n\n # Set parameters and globals related to categories.\n self.globals[\"num_categories\"] = 6\n self.globals[\"category_word_mappings\"] = {'C1': 0, 'C2': 1, 'C3': 2, 'C4': 3, 'BINARY': 4, '': 5}\n self.category_idx_to_word = {0: 'C1', 1: 'C2', 2: 'C3', 3: 'C4', 4: 'BINARY', 5: ''}\n\n # Get image preprocessing.\n self.image_preprocessing = get_value_list_from_dictionary(\n \"image_preprocessing\", self.config,\n 'none | random_affine | random_horizontal_flip | normalize | all'.split(\" | \")\n )\n if 'none' in self.image_preprocessing:\n self.image_preprocessing = []\n if 'all' in self.image_preprocessing:\n self.image_preprocessing = 'random_affine | random_horizontal_flip | normalize'.split(\" | \")\n self.logger.info(\"Applied image preprocessing: {}\".format(self.image_preprocessing))\n\n\n # Get question preprocessing.\n self.question_preprocessing = get_value_list_from_dictionary(\n \"question_preprocessing\", self.config,\n 'none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all'.split(\" | \")\n )\n if 'none' in self.question_preprocessing:\n self.question_preprocessing = []\n if 'all' in self.question_preprocessing:\n self.question_preprocessing = 'lowercase | remove_punctuation | tokenize | remove_stop_words | shuffle_words'.split(\" | \")\n self.logger.info(\"Applied question preprocessing: {}\".format(self.question_preprocessing))\n\n # Get answer preprocessing.\n self.answer_preprocessing = get_value_list_from_dictionary(\n \"answer_preprocessing\", self.config,\n 'none | lowercase | remove_punctuation | tokenize | all'.split(\" | \")\n )\n if 'none' in self.answer_preprocessing:\n self.answer_preprocessing = []\n if 'all' in self.answer_preprocessing:\n self.answer_preprocessing = 'lowercase | remove_punctuation | tokenize '.split(\" | \")\n self.logger.info(\"Applied answer preprocessing: {}\".format(self.answer_preprocessing))\n\n\n # Get the absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get split.\n split = get_value_from_dictionary('split', self.config, \"training,validation,training_validation,test_answers,test\".split(\",\"))\n\n # Set split-dependent data.\n if split == 'training':\n # Training split folder.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Training\")\n # Set source files.\n source_files = [\n os.path.join(split_folder,\"QAPairsByCategory/C1_Modality_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C2_Plane_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C3_Organ_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C4_Abnormality_train.txt\")\n ]\n # Set image folders.\n source_image_folders = [os.path.join(split_folder, 'Train_images')] * 4\n\n # Set the categories associated with each of those files.\n source_categories = [0, 1, 2, 3]\n\n # Filter lists taking into account configuration.\n source_files, source_image_folders, source_categories = self.filter_sources(source_files, source_image_folders, source_categories)\n # Load dataset.\n self.dataset = self.load_dataset(source_files, source_image_folders, source_categories)\n\n elif split == 'validation':\n # Validation split folder.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Validation\")\n\n # Set source files.\n source_files = [\n os.path.join(split_folder,\"QAPairsByCategory/C1_Modality_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C2_Plane_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C3_Organ_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C4_Abnormality_val.txt\")\n ]\n\n # Set image folders.\n source_image_folders = [os.path.join(split_folder, 'Val_images')] * 4\n\n # Set the categories associated with each of those files.\n source_categories = [0, 1, 2, 3]\n\n # Filter lists taking into account configuration.\n source_files, source_image_folders, source_categories = self.filter_sources(source_files, source_image_folders, source_categories)\n # Load dataset.\n self.dataset = self.load_dataset(source_files, source_image_folders, source_categories)\n\n elif split == 'training_validation':\n # This split takes both training and validation and assumes utilization of kFoldWeightedRandomSampler.\n\n # 1. Training split folder.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Training\")\n # Set source files.\n source_files = [\n os.path.join(split_folder,\"QAPairsByCategory/C1_Modality_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C2_Plane_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C3_Organ_train.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C4_Abnormality_train.txt\")\n ]\n # Set image folders.\n source_image_folders = [os.path.join(split_folder, 'Train_images')] * 4\n\n # Set the categories associated with each of those files.\n source_categories = [0, 1, 2, 3]\n\n # Filter lists taking into account configuration.\n training_source_files, training_source_image_folders, training_source_categories = self.filter_sources(source_files, source_image_folders, source_categories)\n\n #2. Validation split folder.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Validation\")\n\n # Set source files.\n source_files = [\n os.path.join(split_folder,\"QAPairsByCategory/C1_Modality_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C2_Plane_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C3_Organ_val.txt\"),\n os.path.join(split_folder,\"QAPairsByCategory/C4_Abnormality_val.txt\")\n ]\n\n # Set image folders.\n source_image_folders = [os.path.join(split_folder, 'Val_images')] * 4\n\n # Set the categories associated with each of those files.\n source_categories = [0, 1, 2, 3]\n\n # Filter lists taking into account configuration.\n valid_source_files, valid_source_image_folders, valid_source_categories = self.filter_sources(source_files, source_image_folders, source_categories)\n\n # 3. Merge lists.\n source_files = [*training_source_files, *valid_source_files]\n source_image_folders = [*training_source_image_folders, *valid_source_image_folders]\n source_categories = [*training_source_categories, *valid_source_categories]\n # Load dataset.\n self.dataset = self.load_dataset(source_files, source_image_folders, source_categories)\n\n elif split == 'test_answers':\n # Test set WITH ANSWERS.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Test\")\n # Set source file.\n source_file = os.path.join(split_folder,\"VQAMed2019_Test_Questions_w_Ref_Answers.txt\")\n # Set image folder.\n source_image_folder = os.path.join(split_folder, 'VQAMed2019_Test_Images')\n self.dataset = self.load_testset_with_answers(source_file, source_image_folder)\n\n else: # \"test\"\n # Test set WITHOUT ANSWERS.\n split_folder = os.path.join(self.data_folder, \"ImageClef-2019-VQA-Med-Test\")\n # Set source file.\n source_file = os.path.join(split_folder,\"VQAMed2019_Test_Questions.txt\")\n # Set image folder.\n source_image_folder = os.path.join(split_folder, 'VQAMed2019_Test_Images')\n self.dataset = self.load_testset_without_answers(source_file, source_image_folder)\n\n # Ok, now we got the whole dataset (for given \"split\").\n self.ix = np.arange(len(self.dataset))\n if self.config[\"import_indices\"] != '':\n # Try to load indices from the file.\n self.ix = np.load(os.path.join(self.app_state.log_dir, self.config[\"import_indices\"]))\n self.logger.info(\"Imported indices from '{}'\".format(os.path.join(self.app_state.log_dir, self.config[\"export_indices\"])))\n else:\n # Ok, check whether we want to shuffle.\n if self.config[\"shuffle_indices\"]:\n np.random.shuffle(self.ix)\n # Export if required.\n if self.config[\"export_indices\"] != '':\n # export indices to file.\n np.save(os.path.join(self.app_state.log_dir, self.config[\"export_indices\"]), self.ix)\n self.logger.info(\"Exported indices to '{}'\".format(os.path.join(self.app_state.log_dir, self.config[\"export_indices\"])))\n\n # Display exemplary sample.\n self.logger.info(\"Exemplary sample 0 ({}):\\n category: {}\\n image_ids: {}\\n question: {}\\n answer: {}\".format(\n self.ix[0],\n self.category_idx_to_word[self.dataset[self.ix[0]][self.key_category_ids]],\n self.dataset[self.ix[0]][self.key_image_ids],\n self.dataset[self.ix[0]][self.key_questions],\n self.dataset[self.ix[0]][self.key_answers]\n ))\n\n # Check if we want the task to calculate and export the weights.\n self.export_sample_weights = self.config[\"export_sample_weights\"]\n if self.export_sample_weights != '':\n self.calculate_and_export_sample_weights(self.export_sample_weights)\n\n\n def output_data_definitions(self):\n \"\"\"\n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n # Add all \"standard\" streams.\n d = {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_image_ids: DataDefinition([-1, 1], [list, str], \"Batch of image names, each being a single word [BATCH_SIZE] x [STRING]\"),\n self.key_category_ids: DataDefinition([-1], [torch.Tensor], \"Batch of target category indices, each being a single index [BATCH_SIZE]\"),\n self.key_category_names: DataDefinition([-1, 1], [list, str], \"Batch of category target names, each being a single word [BATCH_SIZE] x [STRING]\"),\n }\n \n # Return images only when required.\n if self.stream_images:\n d[self.key_images] = DataDefinition([-1, self.depth, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\")\n d[self.key_image_sizes] = DataDefinition([-1, 2], [torch.Tensor], \"Batch of original sizes (height, width) of images [BATCH_SIZE x 2]\")\n\n # Add stream with questions.\n if 'tokenize' in self.question_preprocessing:\n d[self.key_questions] = DataDefinition([-1, -1, 1], [list, list, str], \"Batch of questions, each being a list of words [BATCH_SIZE] x [SEQ_LEN] x [STRING]\")\n else:\n d[self.key_questions] = DataDefinition([-1, 1], [list, str], \"Batch of questions, each being a string consisting of many words [BATCH_SIZE] x [STRING]\")\n\n # Add stream with answers.\n if 'tokenize' in self.answer_preprocessing:\n d[self.key_answers] = DataDefinition([-1, -1, 1], [list, list, str], \"Batch of target answers, each being a list of words [BATCH_SIZE] x [SEQ_LEN] x [STRING]\")\n else:\n d[self.key_answers]= DataDefinition([-1, 1], [list, str], \"Batch of target answers, each being a string consisting of many words [BATCH_SIZE] x [STRING]\")\n return d\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n\n def filter_sources(self, source_files, source_image_folders, source_categories):\n \"\"\"\n Loads the dataset from one or more files.\n\n :param source_files: List of source files.\n\n :param source_image_folders: List of folders containing image files.\n\n :param source_categories: List of categories associated with each of those files. ( unknown)\n\n :return: Tuple consisting of: filtered source_files and filtered source_categories\n \"\"\"\n # Check categories that user want to use.\n use_files = [False] * 4\n categs = {'C1': 0, 'C2': 1, 'C3': 2, 'C4': 3}\n # Parse categories from configuration list.\n loaded_categs = get_value_list_from_dictionary(\"categories\", self.config, ['C1', 'C2', 'C3', 'C4', 'all'])\n for cat in loaded_categs:\n # \"Special\" case.\n if cat == \"all\":\n use_files = [True] * 4\n # Make no sense to continue.\n break\n else:\n use_files[categs[cat]] = True\n # Filter.\n _, source_files, source_image_folders, source_categories = zip(*(filter(lambda x: x[0], zip(use_files, source_files, source_image_folders, source_categories))))\n return source_files, source_image_folders, source_categories\n\n\n def calculate_and_export_sample_weights(self, filename):\n \"\"\"\n Method calculates and export weights associated with samples by looking at distribution of answers.\n\n :param filename: Name of the file (optionally with path) that the sample weights will be saved to.\n \"\"\"\n # 0. Create \"answers dataset\" object for faster computations.\n answers_dataset = []\n for sample in self.dataset:\n if ('tokenize' in self.answer_preprocessing):\n # Need to create one string.\n answers_dataset.append(' '.join(sample[self.key_answers]))\n else:\n answers_dataset.append(sample[self.key_answers])\n\n \n # 1. Iterate over all samples in dataset and create \"answer\" vocabulary.\n answer_to_ix = {}\n for answer in answers_dataset:\n # If new token.\n if answer not in answer_to_ix:\n answer_to_ix[answer] = len(answer_to_ix)\n #print(\"Adding '{}': {}\".format(answer, len(answer_to_ix)-1) )\n\n # 2. Count the samples having the same answer.\n class_sample_count = [0] * len(answer_to_ix)\n for answer in answers_dataset:\n # Increment the adequate class counter.\n class_sample_count[answer_to_ix[answer]] += 1\n\n # 3. Calculate the weights.\n weights = np.asarray([1.0 / count if count > 0 else 0.0 for count in class_sample_count], dtype=np.float64)\n # Normalize to 1.\n sum_w = sum(weights)\n weights = weights/sum_w\n #print(weights)\n\n # 4. Assign weights to samples.\n sample_weights = np.array([weights[answer_to_ix[answer]] for answer in answers_dataset])\n #print(sample_weights)\n #print(len(sample_weights))\n \n # Reorder weights accodring to ix.\n sample_weights_ix = np.array([sample_weights[self.ix[i]] for i in range(len(sample_weights))])\n \n # Process filename.\n (path, name) = os.path.split(filename)\n if path == '':\n # Use default task folder as destination.\n path = self.data_folder\n else:\n path = os.path.expanduser(path)\n\n # Export \"reordered weights\" to file.\n save_nparray_to_csv_file(path, name, sample_weights_ix)\n self.logger.info(\"Generated weights for {} samples and exported them to {}\".format(len(sample_weights_ix), os.path.join(path, name)))\n\n\n def preprocess_text(self, text, lowercase = False, remove_punctuation = False, tokenize = False, remove_stop_words = False):\n \"\"\"\n Function that preprocesses questions/answers as suggested by ImageCLEF VQA challenge organizers:\n * lowercases all words (optional)\n * removes punctuation (optional)\n * removes stop words (optional)\n\n :param text: text to be processed.\n :param lowercase: lowercases text (DEFAULT: False)\n :param remove_punctuation: removes punctuation (DEFAULT: False)\n :param tokenize: tokenizes the text (DEFAULT: False)\n :param remove_stop_words: removes stop words (DEFAULT: False)\n\n :return: Preprocessed and tokenized text (list of strings)\n \"\"\"\n # Lowercase.\n if lowercase:\n text = text.lower()\n\n # Remove punctuation.\n if remove_punctuation:\n # Remove '“' and '”' and '’'!!!\n for char in ['“', '”', '’']:\n text = text.replace(char,' ')\n translator = str.maketrans('', '', string.punctuation)\n text = text.translate(translator)\n\n # If not tokenize - return text.\n if not tokenize:\n return text\n \n # Tokenize.\n text_words = nltk.tokenize.word_tokenize(text)\n\n # If we do not want to remove stop words - return text.\n if not remove_stop_words:\n return text_words\n\n # Perform \"cleansing\".\n stops = set(nltk.corpus.stopwords.words(\"english\"))\n cleansed_words = [word for word in text_words if word not in stops]\n # Return the original text if there are no words left :]\n if len(cleansed_words) == 0:\n return text_words\n\n # Return cleaned text.\n return cleansed_words\n\n def random_remove_stop_words(self, words):\n \"\"\"\n Function removes random stop words, each with 0.5 probability.\n \n :param words: tokenized text\n :return: resulting tokenized text.\n \"\"\"\n\n # Find stop words.\n stops = set(nltk.corpus.stopwords.words(\"english\"))\n stop_words = [False]*len(words)\n for i, word in enumerate(words):\n if word in stops:\n stop_words[i] = True\n #print(stop_words)\n if sum(stop_words) > 0:\n remove_probs = np.random.binomial(1, 0.5, len(words))\n #print(remove_probs)\n result = []\n for word,is_stop,rem_prob in zip(words,stop_words,remove_probs):\n if is_stop and rem_prob:\n # Remove word.\n continue\n # Else: add word.\n result.append(word)\n\n return result\n\n\n def random_shuffle_words(self, words):\n \"\"\"\n Function randomly shuffles, with probability of 0.5, two consecutive words in text.\n \n :param words: tokenized text\n :return: resulting tokenized text.\n \"\"\"\n # Do not shuffle if there are less than 2 words.\n if len(words) < 2:\n return words\n # Shuffle with probability of 0.5.\n if np.random.binomial(1, 0.5, 1):\n return words\n \n # Find words to shuffle - random without replacement.\n shuffled_i = np.random.choice(len(words)-1, )\n indices = [i for i in range(len(words))]\n indices[shuffled_i] = shuffled_i+1\n indices[shuffled_i+1] = shuffled_i\n #print(indices)\n \n # Create resulting table.\n result = [words[indices[i]] for i in range(len(words))]\n\n return result\n\n\n def load_dataset(self, source_files, source_image_folders, source_categories):\n \"\"\"\n Loads the dataset from one or more files.\n\n :param source_files: List of source files.\n\n :param source_image_folders: List of folders containing image files.\n\n :param source_categories: List of categories associated with each of those files. ( unknown)\n \"\"\"\n self.logger.info(\"Loading dataset from files:\\n {}\".format(source_files))\n # Set containing list of tuples.\n dataset = []\n\n # Process files with categories.\n for data_file, image_folder, category in zip(source_files, source_image_folders, source_categories):\n # Set absolute path to file.\n self.logger.info('Loading dataset from {} (category: {})...'.format(data_file, category))\n # Load file content using '|' separator.\n df = pd.read_csv(filepath_or_buffer=data_file, sep='|',header=None,\n names=[self.key_image_ids,self.key_questions,self.key_answers])\n\n # Add tdqm bar.\n t = tqdm.tqdm(total=len(df.index))\n for _, row in df.iterrows():\n # Retrieve question and answer.\n question = row[self.key_questions]\n answer = row[self.key_answers]\n\n # Process question - if required.\n preprocessed_question = self.preprocess_text(\n question,\n 'lowercase' in self.question_preprocessing,\n 'remove_punctuation' in self.question_preprocessing,\n 'tokenize' in self.question_preprocessing,\n 'remove_stop_words' in self.question_preprocessing\n )\n\n # Process answer - if required.\n preprocessed_answer = self.preprocess_text(\n answer,\n 'lowercase' in self.answer_preprocessing,\n 'remove_punctuation' in self.answer_preprocessing,\n 'tokenize' in self.answer_preprocessing,\n False\n )\n\n # Create item \"dictionary\".\n item = {\n # Image name and path leading to it.\n self.key_image_ids: row[self.key_image_ids],\n \"image_folder\": image_folder,\n self.key_questions: preprocessed_question,\n self.key_answers: preprocessed_answer,\n # Add category.\n self.key_category_ids: category\n }\n\n # Preload image.\n if self.preload_images and self.stream_images:\n img, img_size = self.get_image(row[self.key_image_ids], image_folder)\n item[self.key_images] = img\n item[self.key_image_sizes] = img_size\n\n # Add item to dataset.\n dataset.append(item)\n\n t.update()\n t.close()\n\n self.logger.info(\"Loaded dataset consisting of {} samples\".format(len(dataset)))\n # Return the created list.\n return dataset\n\n\n def load_testset_with_answers(self, data_file, image_folder):\n \"\"\"\n Loads the test set with answers.\n\n :param data_file: Source file.\n\n :param image_folder: Folder containing image files.\n\n \"\"\"\n # Set containing list of tuples.\n dataset = []\n category_mapping = {'modality': 0, 'plane': 1, 'organ': 2, 'abnormality': 3}\n\n # Set absolute path to file.\n self.logger.info('Loading test set from {}...'.format(data_file))\n # Load file content using '|' separator.\n df = pd.read_csv(filepath_or_buffer=data_file, sep='|',header=None,\n names=[self.key_image_ids,\"category\",self.key_questions,self.key_answers])\n\n # Add tdqm bar.\n t = tqdm.tqdm(total=len(df.index))\n for _, row in df.iterrows():\n # Retrieve question and answer.\n question = row[self.key_questions]\n answer = row[self.key_answers]\n\n # Process question - if required.\n preprocessed_question = self.preprocess_text(\n question,\n 'lowercase' in self.question_preprocessing,\n 'remove_punctuation' in self.question_preprocessing,\n 'tokenize' in self.question_preprocessing,\n 'remove_stop_words' in self.question_preprocessing\n )\n\n # Process answer - if required.\n preprocessed_answer = self.preprocess_text(\n answer,\n 'lowercase' in self.answer_preprocessing,\n 'remove_punctuation' in self.answer_preprocessing,\n 'tokenize' in self.answer_preprocessing,\n False\n )\n\n # Get category id.\n category_id = category_mapping[row[\"category\"]]\n\n # Create item \"dictionary\".\n item = {\n # Image name and path leading to it.\n self.key_image_ids: row[self.key_image_ids],\n \"image_folder\": image_folder,\n self.key_questions: preprocessed_question,\n self.key_answers: preprocessed_answer,\n # Add category.\n self.key_category_ids: category_id\n }\n\n # Preload image.\n if self.preload_images and self.stream_images:\n img, img_size = self.get_image(row[self.key_image_ids], image_folder)\n item[self.key_images] = img\n item[self.key_image_sizes] = img_size\n\n # Add item to dataset.\n dataset.append(item)\n\n t.update()\n t.close()\n\n self.logger.info(\"Loaded dataset consisting of {} samples\".format(len(dataset)))\n # Return the created list.\n return dataset\n\n\n def load_testset_without_answers(self, data_file, image_folder):\n \"\"\"\n Loads the test set without answers.\n\n :param data_file: Source file.\n\n :param image_folder: Folder containing image files.\n\n \"\"\"\n # Set containing list of tuples.\n dataset = []\n category_id = 5 # \n answer = ''\n\n # Set absolute path to file.\n self.logger.info('Loading test set from {}...'.format(data_file))\n # Load file content using '|' separator.\n df = pd.read_csv(filepath_or_buffer=data_file, sep='|',header=None,\n names=[self.key_image_ids,self.key_questions])\n\n # Add tdqm bar.\n t = tqdm.tqdm(total=len(df.index))\n for _, row in df.iterrows():\n # Retrieve question.\n question = row[self.key_questions]\n\n # Process question - if required.\n preprocessed_question = self.preprocess_text(\n question,\n 'lowercase' in self.question_preprocessing,\n 'remove_punctuation' in self.question_preprocessing,\n 'tokenize' in self.question_preprocessing,\n 'remove_stop_words' in self.question_preprocessing\n )\n\n # Process answer - if required.\n if 'tokenize' in self.answer_preprocessing:\n preprocessed_answer = [answer]\n else:\n preprocessed_answer = answer \n\n # Create item \"dictionary\".\n item = {\n # Image name and path leading to it.\n self.key_image_ids: row[self.key_image_ids],\n \"image_folder\": image_folder,\n self.key_questions: preprocessed_question,\n self.key_answers: preprocessed_answer,\n # Add category.\n self.key_category_ids: category_id\n }\n\n # Preload image.\n if self.preload_images and self.stream_images:\n img, img_size = self.get_image(row[self.key_image_ids], image_folder)\n item[self.key_images] = img\n item[self.key_image_sizes] = img_size\n\n # Add item to dataset.\n dataset.append(item)\n\n t.update()\n t.close()\n\n self.logger.info(\"Loaded dataset consisting of {} samples\".format(len(dataset)))\n # Return the created list.\n return dataset\n\n def get_image(self, img_id, img_folder):\n \"\"\"\n Function loads and returns image along with its size.\n Additionally, it performs all the required transformations.\n\n :param img_id: Identifier of the images.\n :param img_folder: Path to the image.\n\n :return: image (Tensor), image size (Tensor, w,h, both scaled to (0,1>)\n \"\"\"\n\n extension = '.jpg'\n # Load the image.\n img = Image.open(os.path.join(img_folder, img_id + extension))\n # Get its width and height.\n width, height = img.size\n\n image_transformations_list = []\n # Optional.\n if 'random_affine' in self.image_preprocessing:\n rotate = (-45, 80)\n translate = (0.05, 0.25)\n scale = (0.5, 2)\n image_transformations_list.append(transforms.RandomAffine(rotate, translate, scale))\n if 'random_horizontal_flip' in self.image_preprocessing:\n image_transformations_list.append(transforms.RandomHorizontalFlip())\n \n # Add two obligatory transformations.\n image_transformations_list.append(transforms.Resize([self.height,self.width]))\n image_transformations_list.append(transforms.ToTensor())\n\n # Optional normalizastion.\n if 'normalize' in self.image_preprocessing:\n # Use normalization that the pretrained models from TorchVision require.\n image_transformations_list.append(transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))\n\n # Resize the image and transform to Torch Tensor.\n transforms_com = transforms.Compose(image_transformations_list)\n # Apply transformations.\n img = transforms_com(img)\n\n # Get scaled image size.\n img_size = torch.FloatTensor([float(height/self.scale_image_height), float(width/self.scale_image_width)])\n\n # Return image and size.\n return img, img_size\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a single sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: DataStreams({'indices', 'images', 'images_ids','questions', 'answers', 'category_ids', 'image_sizes'})\n \"\"\"\n # Get item.\n item = self.dataset[self.ix[index]]\n\n # Create the resulting sample (data dict).\n data_streams = self.create_data_streams(index)\n\n # Load and stream the image ids.\n img_id = item[self.key_image_ids]\n data_streams[self.key_image_ids] = img_id\n\n # Load the adequate image - only when required.\n if self.stream_images:\n\n if self.preload_images:\n # Use preloaded values.\n img = item[self.key_images] \n img_size = item[self.key_image_sizes] \n else:\n # Load at the very moment.\n img, img_size = self.get_image(img_id, item[\"image_folder\"])\n\n # Image related variables.\n data_streams[self.key_images] = img\n\n # Scale width and height to range (0,1).\n data_streams[self.key_image_sizes] = img_size\n\n # Apply question transformations.\n preprocessed_question = item[self.key_questions]\n if 'tokenize' in self.question_preprocessing:\n # Apply them only if text is tokenized.\n if 'random_remove_stop_words' in self.question_preprocessing:\n preprocessed_question = self.random_remove_stop_words(preprocessed_question)\n\n if 'random_shuffle_words' in self.question_preprocessing:\n preprocessed_question = self.random_shuffle_words(preprocessed_question)\n # Return question.\n data_streams[self.key_questions] = preprocessed_question\n\n # Return answer. \n preprocessed_answer = item[self.key_answers]\n data_streams[self.key_answers] = preprocessed_answer\n\n # Question category related variables.\n # Check if this is binary question.\n if self.predict_yes_no(item[self.key_questions]):\n data_streams[self.key_category_ids] = 4 # Binary.\n data_streams[self.key_category_names] = self.category_idx_to_word[4]\n else:\n data_streams[self.key_category_ids] = item[self.key_category_ids]\n data_streams[self.key_category_names] = self.category_idx_to_word[item[self.key_category_ids]]\n\n # Return sample.\n return data_streams\n\n def predict_yes_no(self, qtext):\n \"\"\"\n Determines whether this is binary (yes/no) type of question.\n \"\"\"\n yes_no_starters = ['is','was','are','does']\n if 'tokenize' not in self.question_preprocessing:\n qtext = qtext.split(' ')\n first_token = qtext[0]\n if first_token in yes_no_starters and ('or' not in qtext):\n return True\n return False\n\n def collate_fn(self, batch):\n \"\"\"\n Combines a list of DataStreams (retrieved with :py:func:`__getitem__`) into a batch.\n\n :param batch: list of individual samples to combine\n :type batch: list\n\n :return: DataStreams({'indices', 'images', 'images_ids','questions', 'answers', 'category_ids', 'image_sizes'})\n\n \"\"\"\n # Collate indices.\n data_streams = self.create_data_streams([sample[self.key_indices] for sample in batch])\n\n # Stack images.\n data_streams[self.key_image_ids] = [item[self.key_image_ids] for item in batch]\n if self.stream_images:\n data_streams[self.key_images] = torch.stack([item[self.key_images] for item in batch]).type(torch.FloatTensor)\n data_streams[self.key_image_sizes] = torch.stack([item[self.key_image_sizes] for item in batch]).type(torch.FloatTensor)\n\n # Collate lists/lists of lists.\n data_streams[self.key_questions] = [item[self.key_questions] for item in batch]\n data_streams[self.key_answers] = [item[self.key_answers] for item in batch]\n\n # Stack categories.\n data_streams[self.key_category_ids] = torch.tensor([item[self.key_category_ids] for item in batch])\n data_streams[self.key_category_names] = [item[self.key_category_names] for item in batch]\n\n # Return collated dict.\n return data_streams\n" }, { "path": "ptp/components/tasks/image_to_class/__init__.py", "content": "#from .cifar10 import CIFAR10\nfrom .cifar_100 import CIFAR100\nfrom .mnist import MNIST\nfrom .simple_molecules import SimpleMolecules\n\n__all__ = [\n 'CIFAR100',\n 'MNIST',\n 'SimpleMolecules',\n ]\n" }, { "path": "ptp/components/tasks/image_to_class/cifar_100.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport torch\nfrom torchvision import datasets, transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\nclass CIFAR100(Task):\n \"\"\"\n Classic CIFAR-100 image classification task.\n\n Reference page: http://www.cs.toronto.edu/~kriz/cifar.html\n\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the task.\n\n .. warning::\n\n Resizing images might cause a significant slow down in batch generation.\n\n :param name: Task name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n \"\"\"\n\n # Call base class constructors.\n super(CIFAR100, self).__init__(name, CIFAR100, config)\n\n # Get default key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n # Targets.\n self.key_coarse_targets = self.stream_keys[\"coarse_targets\"]\n self.key_fine_targets = self.stream_keys[\"fine_targets\"]\n # Streams returning targets as words (labels).\n self.key_coarse_labels = self.stream_keys[\"coarse_labels\"]\n self.key_fine_labels = self.stream_keys[\"fine_labels\"]\n\n # Get absolute path.\n data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Retrieve parameters from the dictionary.\n self.use_train_data = self.config['use_train_data']\n\n # Add transformations depending on the resizing option.\n if 'resize_image' in self.config:\n # Check the desired size.\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n\n # Up-scale and transform to tensors.\n transform = transforms.Compose([transforms.Resize((self.height, self.width)), transforms.ToTensor()])\n\n self.logger.warning('Upscaling the images to [{}, {}]. Slows down batch generation.'.format(\n self.width, self.height))\n\n else:\n # Default CIFAR-100 settings.\n self.width = 32\n self.height = 32\n # Simply turn to tensor.\n transform = transforms.Compose([transforms.ToTensor()])\n\n # Load the dataset. (PROBLEM WITH COARSE-TO-FINE LABEL MAPPING!)\n self.dataset = datasets.CIFAR100(root=data_folder, train=self.use_train_data, download=True, transform=transform)\n\n # Process labels.\n all_labels = {\"aquatic_mammals\": \"beaver, dolphin, otter, seal, whale\".split(\", \"),\n \"fish\": \"aquarium_fish, flatfish, ray, shark, trout\".split(\", \"),\n \"flowers\": \"orchid, poppy, rose, sunflower, tulip\".split(\", \"),\n \"food_containers\": \"bottle, bowl, can, cup, plate\".split(\", \"),\n \"fruit_and_vegetables\": \"apple, mushroom, orange, pear, sweet_pepper\".split(\", \"),\n \"household_electrical_devices\": \"clock, keyboard, lamp, telephone, television\".split(\", \"),\n \"household_furniture\": \"bed, chair, couch, table, wardrobe\".split(\", \"),\n \"insects\": \"bee, beetle, butterfly, caterpillar, cockroach\".split(\", \"),\n \"large_carnivores\": \"bear, leopard, lion, tiger, wolf\".split(\", \"),\n \"large_man-made_outdoor_things\": \"bridge, castle, house, road, skyscraper\".split(\", \"),\n \"large_natural_outdoor_scenes\": \"cloud, forest, mountain, plain, sea\".split(\", \"),\n \"large_omnivores_and_herbivores\": \"camel, cattle, chimpanzee, elephant, kangaroo\".split(\", \"),\n \"medium-sized_mammals\": \"fox, porcupine, possum, raccoon, skunk\".split(\", \"),\n \"non-insect_invertebrates\": \"crab, lobster, snail, spider, worm\".split(\", \"),\n \"people\": \"baby, boy, girl, man, woman\".split(\", \"),\n \"reptiles\": \"crocodile, dinosaur, lizard, snake, turtle\".split(\", \"),\n \"small_mammals\": \"hamster, mouse, rabbit, shrew, squirrel\".split(\", \"),\n \"trees\": \"maple_tree, oak_tree, palm_tree, pine_tree, willow_tree\".split(\", \"),\n \"vehicles_1\": \"bicycle, bus, motorcycle, pickup_truck, train\".split(\", \"),\n \"vehicles_2\": \"lawn_mower, rocket, streetcar, tank, tractor\".split(\", \")}\n\n coarse_word_to_ix = {}\n fine_to_coarse_mapping = {}\n fine_labels = []\n for coarse_id, (key, values) in enumerate(all_labels.items()):\n # Add mapping from coarse category name to coarse id.\n coarse_word_to_ix[key] = coarse_id\n # Add mappings from fine category names to coarse id.\n for value in values:\n fine_to_coarse_mapping[value] = coarse_id\n # Add values to list of fine labels.\n fine_labels.extend(values)\n\n # Sort fine labels.\n fine_labels = sorted(fine_labels)\n\n # Generate fine word mappings.\n fine_word_to_ix = {fine_labels[i]: i for i in range(len(fine_labels))}\n # Export fine word mappings to globals.\n self.globals[\"fine_label_word_mappings\"] = fine_word_to_ix\n # Reverse mapping - for labels.\n self.fine_ix_to_word = {value: key for (key, value) in fine_word_to_ix.items()}\n\n # Export coarse word mappings to globals.\n self.globals[\"coarse_label_word_mappings\"] = coarse_word_to_ix\n # Reverse mapping - for labels.\n self.coarse_ix_to_word = {value: key for (key, value) in coarse_word_to_ix.items()}\n\n # Create fine to coarse id mapping.\n self.fine_to_coarse_id_mapping = {}\n for fine_label, fine_id in fine_word_to_ix.items():\n self.fine_to_coarse_id_mapping[fine_id] = fine_to_coarse_mapping[fine_label]\n #print(\" {} ({}) : {} \".format(fine_label, fine_id, self.coarse_ix_to_word[fine_to_coarse_mapping[fine_label]]))\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"num_coarse_classes\"] = len(coarse_word_to_ix)\n self.globals[\"num_fine_classes\"] = len(fine_labels)\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_depth\"] = 3\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_inputs: DataDefinition([-1, 3, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\"),\n self.key_coarse_targets: DataDefinition([-1], [torch.Tensor], \"Batch of coarse targets, each being a single index [BATCH_SIZE]\"),\n self.key_coarse_labels: DataDefinition([-1, 1], [list, str], \"Batch of coarse labels, each being a single word [BATCH_SIZE] x [STRING]\"),\n self.key_fine_targets: DataDefinition([-1], [torch.Tensor], \"Batch of fine targets, each being a single index [BATCH_SIZE]\"),\n self.key_fine_labels: DataDefinition([-1, 1], [list, str], \"Batch of fine labels, each being a single word [BATCH_SIZE] x [STRING]\")\n }\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'images','targets'})``, with:\n\n - images: Image, resized if ``self.resize`` is set,\n - targets: Index of the target class\n \"\"\"\n # Get image and fine label id.\n image, fine_target = self.dataset.__getitem__(index)\n\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_inputs] = image\n # Targets.\n data_streams[self.key_coarse_targets] = self.fine_to_coarse_id_mapping[fine_target]\n data_streams[self.key_fine_targets] = fine_target\n # Labels.\n data_streams[self.key_coarse_labels] = self.coarse_ix_to_word[self.fine_to_coarse_id_mapping[fine_target]]\n data_streams[self.key_fine_labels] = self.fine_ix_to_word[fine_target]\n\n #print(data_streams)\n return data_streams\n" }, { "path": "ptp/components/tasks/image_to_class/mnist.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta, Younes Bouhadjar, Vincent Marois\"\n\nimport os\nimport torch\nfrom torchvision import datasets, transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\nclass MNIST(Task):\n \"\"\"\n Classic MNIST digit classification task.\n\n Please see reference here: http://yann.lecun.com/exdb/mnist/\n\n .. warning::\n\n The dataset is not originally split into a training set, validation set and test set; only\\\n training and test set. \n \n In order to split training into training and validation sets please set distinctive index ranges in configuration file.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the MNIST task.\n\n .. warning::\n\n Resizing images might cause a significant slow down in batch generation.\n\n :param name: Task name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n \"\"\"\n\n # Call base class constructors.\n super(MNIST, self).__init__(name, MNIST, config)\n\n # Get default key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_targets = self.stream_keys[\"targets\"]\n # Stream returning targets as words.\n self.key_labels = self.stream_keys[\"labels\"]\n\n # Get absolute path.\n data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Retrieve parameters from the dictionary.\n self.use_train_data = self.config['use_train_data']\n\n # Add transformations depending on the resizing option.\n if 'resize_image' in self.config:\n # Check the desired size.\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n\n # Up-scale and transform to tensors.\n transform = transforms.Compose([transforms.Resize((self.height, self.width)), transforms.ToTensor()])\n\n self.logger.warning('Upscaling the images to [{}, {}]. Slows down batch generation.'.format(\n self.width, self.height))\n\n else:\n # Default MNIST settings.\n self.width = 28\n self.height = 28\n # Simply turn to tensor.\n transform = transforms.Compose([transforms.ToTensor()])\n\n # load the dataset\n self.dataset = datasets.MNIST(root=data_folder, train=self.use_train_data, download=True,\n transform=transform)\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"num_classes\"] = 10\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_depth\"] = 1\n\n # Class names.\n labels = 'Zero One Two Three Four Five Six Seven Eight Nine'.split(' ')\n # Export to globals.\n word_to_ix = {labels[i]: i for i in range(10)}\n self.globals[\"label_word_mappings\"] = word_to_ix\n # Reverse mapping - for labels.\n self.ix_to_word = {value: key for (key, value) in word_to_ix.items()}\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_inputs: DataDefinition([-1, 1, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\"),\n self.key_targets: DataDefinition([-1], [torch.Tensor], \"Batch of targets, each being a single index [BATCH_SIZE]\"),\n self.key_labels: DataDefinition([-1, 1], [list, str], \"Batch of targets, each being a single word [BATCH_SIZE] x [STRING]\")\n }\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'images','targets'})``, with:\n\n - images: Image, resized if ``self.resize`` is set,\n - targets: Index of the target class\n \"\"\"\n # Get image and target.\n img, target = self.dataset.__getitem__(index)\n \n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_inputs] = img\n data_streams[self.key_targets] = target\n data_streams[self.key_labels] = self.ix_to_word[target.item()]\n return data_streams\n" }, { "path": "ptp/components/tasks/image_to_class/simple_molecules.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\nimport csv\nfrom PIL import Image\n\nimport torch\nfrom torchvision import transforms\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\nfrom ptp.configuration.config_parsing import get_value_from_dictionary\nfrom ptp.configuration.configuration_error import ConfigurationError\n\n\nclass SimpleMolecules(Task):\n \"\"\"\n Simple molecule classification task.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes the task.\n\n .. warning::\n\n Resizing images might cause a significant slow down in batch generation.\n\n :param name: Task name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n \"\"\"\n\n # Call base class constructors.\n super(SimpleMolecules, self).__init__(name, SimpleMolecules, config)\n\n # Get default key mappings.\n self.key_images = self.stream_keys[\"images\"]\n self.key_targets = self.stream_keys[\"targets\"]\n # Stream returning targets as words.\n self.key_labels = self.stream_keys[\"labels\"]\n\n # Add transformations depending on the resizing option.\n if 'resize_image' in self.config:\n # Check the desired size.\n if len(self.config['resize_image']) != 2:\n self.logger.error(\"'resize_image' field must contain 2 values: the desired height and width\")\n exit(-1)\n\n # Output image dimensions.\n self.height = self.config['resize_image'][0]\n self.width = self.config['resize_image'][1]\n\n # Up-scale and transform to tensors.\n self.image_transforms = transforms.Compose([transforms.Resize((self.height, self.width)), transforms.ToTensor()])\n\n self.logger.warning('Upscaling the images to [{}, {}]. Slows down batch generation.'.format(\n self.width, self.height))\n\n else:\n # Default settings.\n self.width = 875\n self.height = 875\n # Simply turn to tensor.\n self.image_transforms = transforms.Compose([transforms.ToTensor()])\n\n # Get image depth.\n self.image_depth = self.config[\"image_depth\"]\n\n # Set global variables - all dimensions ASIDE OF BATCH.\n self.globals[\"num_classes\"] = 10\n self.globals[\"image_width\"] = self.width\n self.globals[\"image_height\"] = self.height\n self.globals[\"image_depth\"] = self.image_depth\n\n # Class names.\n labels = 'Zero One Two Three Four Five Six Seven Eight Nine'.split(' ')\n # Export to globals.\n word_to_ix = {labels[i]: i for i in range(10)}\n self.globals[\"label_word_mappings\"] = word_to_ix\n # Reverse mapping - for labels.\n self.ix_to_word = {value: key for (key, value) in word_to_ix.items()}\n\n # Get the absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get the split.\n split = get_value_from_dictionary('split', self.config, \"training | validation | test | test_mirror_blur | test_on_grid | test_handwritten\".split(\" | \"))\n\n # Set split-dependent data.\n if split == 'training':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Dist_Labels_Set0.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Dist\")\n\n elif split == 'validation':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Dist_Labels_Set1.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Dist\")\n\n elif split == 'test':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Dist_Labels_Set2.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Dist\")\n\n elif split == 'test_mirror_blur':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Test1_Labels.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Test1\")\n\n elif split == 'test_on_grid':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Test2_Labels.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Test2\")\n\n elif split == 'test_handwritten':\n # Training split folder and file with data question.\n data_file = os.path.join(self.data_folder, 'ChemDATA_A_Test3_Labels.tsv')\n self.image_folder = os.path.join(self.data_folder, \"ChemDATA_A_Test3\")\n\n else: \n raise ConfigurationError(\"Split {} not supported yet\".format(split))\n\n # Load dataset.\n self.dataset = self.load_dataset(data_file)\n \n # Display exemplary sample.\n i = 0\n sample = self.dataset[i]\n\n self.logger.info(\"Exemplary sample {}:\\n image_ids: {}\\n class {}\".format(\n i,\n sample[1],\n sample[0]\n ))\n\n\n def load_dataset(self, source_data_file):\n \"\"\"\n Loads the dataset from source file\n\n :param source_data_file: csv file containing label-image filename pairs.\n\n \"\"\"\n self.logger.info(\"Loading dataset from:\\n {}\".format(source_data_file))\n dataset = []\n\n with open(source_data_file, 'r') as f:\n self.logger.info(\"Loading samples from '{}'...\".format(source_data_file))\n dataset = list(csv.reader(f, delimiter='\\t'))\n\n self.logger.info(\"Loaded split consisting of {} samples\".format(len(dataset)))\n return dataset\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.dataset)\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_images: DataDefinition([-1, self.image_depth, self.height, self.width], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\"),\n self.key_targets: DataDefinition([-1], [torch.Tensor], \"Batch of targets, each being a single index [BATCH_SIZE]\"),\n self.key_labels: DataDefinition([-1, 1], [list, str], \"Batch of targets, each being a single word [BATCH_SIZE] x [STRING]\")\n }\n\n def get_image(self, img_id):\n \"\"\"\n Function loads and returns image along with its size.\n Additionally, it performs all the required transformations.\n\n :param img_id: Identifier of the images.\n :param img_folder: Path to the image.\n\n :return: image (Tensor)\n \"\"\"\n\n # Load the image.\n img = Image.open(os.path.join(self.image_folder, img_id + '.png'))\n \n # This may be required by some models e.g. the ones pretrained on ImageNet.\n if self.image_depth == 3:\n img = img.convert('RGB')\n\n # Apply transformations.\n img = self.image_transforms(img)\n\n # Return image.\n return img\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'images','targets'})``, with:\n\n - images: Image, resized if ``self.resize`` is set,\n - targets: Index of the target class\n \"\"\"\n # Get image and target.\n (label, img_id) = self.dataset[index]\n \n # Load the image.\n img = self.get_image(img_id)\n\n target = int(label)\n\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_images] = img\n data_streams[self.key_targets] = target\n data_streams[self.key_labels] = label\n return data_streams\n" }, { "path": "ptp/components/tasks/task.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019-2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\nfrom torch.utils.data import Dataset\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_streams import DataStreams\n\n\nclass Task(Component, Dataset):\n \"\"\"\n Class representing base class for all Tasks.\n\n Inherits from :py:class:`torch.utils.data.Dataset` as all subclasses will represent a task with an associated dataset,\\\n and the `worker` will use :py:class:`torch.utils.data.DataLoader` to generate batches.\n\n Implements features & attributes used by all subclasses.\n\n \"\"\"\n\n def __init__(self, name, class_type, config):\n \"\"\"\n Initializes task object:\n - calls base class constructors.\n - sets key_indices variable (used for storing indices of samples)\n\n >>> self.key_indices = self.mapkey(\"indices\")\n\n - sets empry curriculim learning configuration\n\n >>> self.curriculum_config = {}\n \n :param name: Task name.\n :type name: str\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n .. note::\n\n It is likely to encounter a case where the model needs a parameter value only known when the task has been\n instantiated, like the size of a vocabulary set or the number of marker bits.\n\n The user can pass those values in this app_state. All objects will be able to access it later:\n\n >>> self.app_state[\"new_global_value\"] = 1 # Sets global value.\n >>> val = self.app_state[\"new_global_value\" # Gets global value.\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, class_type, config)\n Dataset.__init__(self)\n\n # Get key mappings for indices.\n self.key_indices = self.stream_keys[\"indices\"]\n\n # Empty curriculum learning config - for now.\n self.curriculum_config = {}\n\n\n def summarize_io(self, priority = -1):\n \"\"\"\n Summarizes the task by showing its name, type and output definitions.\n\n :param priority: Task priority (DEFAULT: -1)\n\n :return: Summary as a str.\n\n \"\"\"\n summary_str = \" + {} ({}) [{}]\\n\".format(self.name, type(self).__name__, priority)\n # Get outputs.\n summary_str += ' Outputs:\\n' \n for key,value in self.output_data_definitions().items():\n summary_str += ' {}: {}, {}, {}\\n'.format(key, value.dimensions, value.types, value. description)\n return summary_str\n\n def __call__(self, data_streams):\n \"\"\"\n Method responsible for processing the data dict. Empty for all task-derived classes.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing both input data to be proces and that will be extended by the results.\n \"\"\"\n pass\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n As there is assumption made (tasks do not accept inputs) it returns empty dictionary.\n\n :return: Empty dictionary.\n \"\"\"\n return {}\n\n\n def create_data_streams(self, index, data_definitions = None):\n \"\"\"\n Returns a :py:class:`ptp.utils.DataStreams` object with keys created on the \\\n task data_definitions and empty values (None).\n\n :param data_definitions: Data definitions that will be used (DEFAULT: None, meaninng that self.output_data_definitions() will be used)\n\n :return: new :py:class:`ptp.utils.DataStreams` object.\n \"\"\"\n # Use self.output_data_definitions() if required\n data_definitions = data_definitions if data_definitions is not None else self.output_data_definitions()\n # Add index - just in case. This key is required!\n if self.key_indices not in data_definitions:\n data_definitions[self.key_indices] = None\n data_streams = DataStreams({key: None for key in data_definitions.keys()})\n # Set index.\n data_streams[self.key_indices] = index\n return data_streams\n\n\n def collate_fn(self, batch):\n \"\"\"\n Generates a batch of samples from a list of individuals samples retrieved by :py:func:`__getitem__`.\n\n The method calls :py:func:`torch.utils.data.dataloader.default_collate` for every item in data_streams !\n \n .. note::\n\n This base :py:func:`collate_fn` method only calls the default \\\n :py:func:`torch.utils.data.dataloader.default_collate`, as it can handle several cases \\\n (mainly tensors, numbers, dicts and lists).\n\n If your dataset can yield variable-length samples within a batch, or generate batches `on-the-fly`\\\n , or possesses another `non regular` characteristic, it is most likely that you will need to \\\n override this default :py:func:`collate_fn`.\n\n\n :param batch: List of :py:class:`ptp.utils.DataStreams` retrieved by :py:func:`__getitem__`, each containing \\\n tensors, numbers, dicts or lists.\n :type batch: list\n\n :return: DataStreams containing the created batch.\n\n \"\"\"\n return DataStreams({key: torch.utils.data.dataloader.default_collate([sample[key] for sample in batch]) for key in batch[0]})\n\n\n def initialize_epoch(self, epoch):\n \"\"\"\n Function called to initialize a new epoch.\n\n .. note::\n\n Empty - To be redefined in inheriting classes.\n\n :param epoch: current epoch\n :type epoch: int\n \"\"\"\n pass\n\n def finalize_epoch(self, epoch):\n \"\"\"\n Function called at the end of an epoch to finalize it.\n\n .. note::\n\n Empty - To be redefined in inheriting classes.\n\n :param epoch: current epoch\n :type epoch: int\n \"\"\"\n pass\n\n\n def curriculum_learning_initialize(self, curriculum_config):\n \"\"\"\n Initializes curriculum learning - simply saves the curriculum config.\n\n .. note::\n\n This method can be overwritten in the derived classes.\n\n\n :param curriculum_config: Interface to parameters accessing curriculum learning view of the registry tree.\n :type param: :py:class:`ptp.configuration.ConfigInterface`\n\n\n \"\"\"\n # Save config.\n self.curriculum_config = curriculum_config\n\n\n def curriculum_learning_update_params(self, episode, epoch):\n \"\"\"\n Updates task parameters according to curriculum learning.\n\n .. note::\n\n This method can be overwritten in the derived classes.\n\n :param episode: Number of the current episode.\n :type episode: int\n\n :param epoch: Number of the current epoch.\n :type epoch: int\n\n :return: True informing that Curriculum Learning wasn't active at all (i.e. is finished).\n\n \"\"\"\n\n return True\n" }, { "path": "ptp/components/tasks/text_to_class/__init__.py", "content": "from .dummy_language_identification import DummyLanguageIdentification\nfrom .language_identification import LanguageIdentification\nfrom .wily_language_identification import WiLYLanguageIdentification\nfrom .wily_ngram_language_modeling import WiLYNGramLanguageModeling\n\n__all__ = [\n 'DummyLanguageIdentification',\n 'LanguageIdentification',\n 'WiLYLanguageIdentification',\n 'WiLYNGramLanguageModeling'\n ]\n" }, { "path": "ptp/components/tasks/text_to_class/dummy_language_identification.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom .language_identification import LanguageIdentification\n\nimport os\n\nimport ptp.components.mixins.io as io\n\n\nclass DummyLanguageIdentification(LanguageIdentification):\n \"\"\"\n Simple Language identification (classification) task.\n Data taken from the _example.\n\n .. _example: https://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes the task object. Calls base constructor and generates the files, if not present.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n LanguageIdentification.__init__(self, name, DummyLanguageIdentification, config) \n\n # Get absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Generate the dataset (can be turned off).\n filenames = [\"x_training.txt\", \"y_training.txt\", \"x_test.txt\", \"y_test.txt\"]\n if self.config['regenerate'] or not io.check_files_existence(self.data_folder, filenames):\n self.generate_dataset()\n\n # Select set.\n if self.config['use_train_data']:\n inputs_file = \"x_training.txt\"\n targets_file = \"y_training.txt\"\n else:\n inputs_file = \"x_test.txt\"\n targets_file = \"y_test.txt\"\n\n # Load files.\n self.inputs = io.load_string_list_from_txt_file(self.data_folder, inputs_file)\n self.targets = io.load_string_list_from_txt_file(self.data_folder, targets_file)\n\n # Assert that they are equal in size!\n assert len(self.inputs) == len(self.targets), \"Number of inputs loaded from {} not equal to number of targets loaded from {}!\".format(inputs_file, targets_file)\n\n\n def generate_dataset(self):\n \"\"\"\n Method generates dummy dataset for language identification, few (sentence-language) pairs, training and text sets.\n \"\"\"\n self.logger.info(\"Generating dummy dataset in {}\".format(self.data_folder))\n\n # \"Training\" set.\n x_training_data = [\n \"me gusta comer en la cafeteria\",\n \"Give it to me\", \n \"No creo que sea una buena idea\",\n \"No it is not a good idea to get lost at sea\"]\n io.save_string_list_to_txt_file(self.data_folder, 'x_training.txt', x_training_data)\n\n y_training_data = [\n \"SPANISH\",\n \"ENGLISH\",\n \"SPANISH\",\n \"ENGLISH\"]\n io.save_string_list_to_txt_file(self.data_folder, 'y_training.txt', y_training_data)\n\n # \"Test\" set.\n x_test_data = [\n \"Yo creo que si\",\n \"it is lost on me\"]\n io.save_string_list_to_txt_file(self.data_folder, 'x_test.txt', x_test_data)\n\n y_test_data = [\n \"SPANISH\",\n \"ENGLISH\"]\n io.save_string_list_to_txt_file(self.data_folder, 'y_test.txt', y_test_data)\n\n self.logger.info(\"Initialization successfull\")\n" }, { "path": "ptp/components/tasks/text_to_class/language_identification.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass LanguageIdentification(Task):\n \"\"\"\n Language identification (classification) task.\n \"\"\"\n\n def __init__(self, name, class_type, config):\n \"\"\"\n Initializes task object. Calls base constructor.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n Task.__init__(self, name, class_type, config)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_targets = self.stream_keys[\"targets\"]\n\n # Set empty inputs and targets.\n self.inputs = []\n self.targets = []\n\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_inputs: DataDefinition([-1, 1], [list, str], \"Batch of sentences, each being a single string (many words) [BATCH_SIZE x SENTENCE]\"),\n self.key_targets: DataDefinition([-1, 1], [list, str], \"Batch of targets, each being a single label (word) BATCH_SIZE x WORD]\")\n }\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.inputs)\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'inputs','targets'})``\n\n \"\"\"\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_inputs] = self.inputs[index]\n data_streams[self.key_targets] = self.targets[index]\n return data_streams\n" }, { "path": "ptp/components/tasks/text_to_class/wily_language_identification.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\n\nimport ptp.components.mixins.io as io\nfrom .language_identification import LanguageIdentification\n\n\nclass WiLYLanguageIdentification(LanguageIdentification):\n \"\"\"\n Language identification (classification) task.\n Using WiLI-2018 benchmark _dataset taken from the paper: Thoma, Martin. \"The WiLI benchmark dataset for written language identification.\" arXiv preprint arXiv:1801.07779 (2018). (_arxiv)\n\n The dataset contains sentences from 235 languages.\n\n .. _dataset: https://zenodo.org/record/841984\n .. _arxiv: https://arxiv.org/abs/1801.07779\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes task object. Calls base constructor. Downloads the dataset if not present and loads the adequate files depending on the mode.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n LanguageIdentification.__init__(self, name, WiLYLanguageIdentification, config) \n\n # Get absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Generate the dataset (can be turned off).\n filenames = [\"x_train.txt\", \"y_train.txt\", \"x_test.txt\", \"y_test.txt\"]\n if not io.check_files_existence(self.data_folder, filenames):\n # Download and unpack.\n url = \"https://zenodo.org/record/841984/files/wili-2018.zip?download=1\"\n zipfile_name = \"wili-2018.zip\"\n io.download_extract_zip_file(self.logger, self.data_folder, url, zipfile_name)\n\n # Select set.\n if self.config['use_train_data']:\n inputs_file = \"x_train.txt\"\n targets_file = \"y_train.txt\"\n else:\n inputs_file = \"x_test.txt\"\n targets_file = \"y_test.txt\"\n\n # Load files.\n self.inputs = io.load_string_list_from_txt_file(self.data_folder, inputs_file)\n self.targets = io.load_string_list_from_txt_file(self.data_folder, targets_file)\n\n # Assert that they are equal in size!\n assert len(self.inputs) == len(self.targets), \"Number of inputs loaded from {} not equal to number of targets loaded from {}!\".format(inputs_file, targets_file)\n" }, { "path": "ptp/components/tasks/text_to_class/wily_ngram_language_modeling.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\n\nimport ptp.components.mixins.io as io\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass WiLYNGramLanguageModeling(Task):\n \"\"\"\n N-gram Language Modeling task.\n By default it is using sentences from the WiLI benchmark _dataset taken from the paper: Thoma, Martin. \"The WiLI benchmark dataset for written language identification.\" arXiv preprint arXiv:1801.07779 (2018). (_arxiv)\n\n .. _dataset: https://zenodo.org/record/841984\n .. _arxiv: https://arxiv.org/abs/1801.07779\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n Initializes task object. Calls base constructor.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructors of parent classes.\n Task.__init__(self, name, WiLYNGramLanguageModeling, config) \n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_targets = self.stream_keys[\"targets\"]\n\n # Get absolute path.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get size of context.\n self.context = self.config['context']\n\n # Select set.\n if self.config['use_train_data']:\n inputs_file = \"x_train.txt\"\n ngrams_file = \"ngrams_train.txt\"\n else:\n inputs_file = \"x_test.txt\"\n ngrams_file = \"ngrams_test.txt\"\n\n # Check if we can load ngrams.\n if not io.check_file_existence(self.data_folder, ngrams_file):\n # Sadly not, we have to generate them.\n if not io.check_file_existence(self.data_folder, inputs_file):\n # Even worst - we have to download wily.\n url = \"https://zenodo.org/record/841984/files/wili-2018.zip?download=1\"\n zipfile_name = \"wili-2018.zip\"\n io.download_extract_zip_file(self.logger, self.data_folder, url, zipfile_name)\n\n\n # Load file.\n inputs = io.load_string_list_from_txt_file(self.data_folder, inputs_file)\n\n self.logger.info(\"Please wait, generating n-grams...\")\n self.ngrams_sent = []\n # Now we have to split sentencese into n-grams.\n for sentence in inputs:\n # Split sentence into words.\n words = sentence.split()\n \n # Build a list of ngrams.\n for i in range(len(words) - self.context):\n ngram = [words[j] for j in range(i, i+1+self.context)]\n self.ngrams_sent.append(' '.join(ngram))\n\n # Assert that they are any ngrams there!\n assert len(self.ngrams_sent) > 0, \"Number of n-grams generated on the basis of '{}' must be greater than 0!\".format(inputs_file)\n # Done.\n self.logger.info(\"Generated {} n-grams, example:\\n{}\".format(len(self.ngrams_sent), self.ngrams_sent[0]))\n\n self.logger.info(\"Saving {} n-grams to file '{}'\".format(len(self.ngrams_sent), ngrams_file))\n # N-grams generated, save them to file.\n io.save_string_list_to_txt_file(self.data_folder, ngrams_file, self.ngrams_sent)\n else:\n self.logger.info(\"Please wait, loading n-grams from file '{}'\".format(ngrams_file))\n # Load file.\n self.ngrams_sent = io.load_string_list_from_txt_file(self.data_folder, ngrams_file)\n\n # Assert that they are equal in size!\n assert len(self.ngrams_sent) > 0, \"Number of n-grams loaded from {} must be greater than 0!\".format(ngrams_file)\n # Done.\n self.logger.info(\"Loaded {} n-grams, example:\\n{}\".format(len(self.ngrams_sent), self.ngrams_sent[0]))\n \n # Split words in n-grams.\n self.ngrams = [ngram.split() for ngram in self.ngrams_sent]\n \n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n #self.key_inputs: DataDefinition([-1, self.context, 1], [list, list, str], \"Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [CONTEXT_SIZE] x [WORD]\"),\n self.key_inputs: DataDefinition([-1, 1], [list, str], \"Batch of sentences, each being a context consisint of several words [BATCH_SIZE] x [string CONTEXT_SIZE * WORD]\"),\n self.key_targets: DataDefinition([-1, 1], [list, str], \"Batch of targets, each being a single word [BATCH_SIZE] x [WORD]\")\n }\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return len(self.ngrams)\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'inputs','targets'})``\n\n \"\"\"\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_inputs] = ' '.join(self.ngrams[index][:self.context])\n data_streams[self.key_targets] = self.ngrams[index][-1] # Last word\n #print(\"task: context = {} target = {}\".format(data_streams[self.key_inputs], data_streams[self.key_targets]))\n return data_streams\n" }, { "path": "ptp/components/tasks/text_to_text/__init__.py", "content": "from .wikitext_language_modeling import WikiTextLanguageModeling\nfrom .translation_pairs import TranslationPairs\n\n__all__ = [\n 'WikiTextLanguageModeling',\n 'TranslationPairs'\n ]\n" }, { "path": "ptp/components/tasks/text_to_text/translation_pairs.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman\"\n\nimport os\nimport random\nimport tempfile\nimport unicodedata\nimport re\n\nfrom nltk.tokenize import WhitespaceTokenizer\n\nimport ptp.components.mixins.io as io\nfrom ptp.configuration import ConfigurationError\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass TranslationPairs(Task):\n \"\"\"\n Bilingual sentence pairs from http://www.manythings.org/anki/.\n Only some pairs are included here, but many more are available on the website.\n Will download the requested language pair if necessary, normalize and tokenize the sentences, and will cut the data into train, valid, test sets.\n\n Resulting tokens that are shorter than the specified length are then passed to samples (source/target) as list of tokens (set by the user in configuration file).\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n The init method downloads the required files, loads the file associated with a given subset (train/valid/test), \n concatenates all sencentes and tokenizes them using NLTK's WhitespaceTokenizer.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructor of parent classes.\n Task.__init__(self, name, TranslationPairs, config) \n\n # Set streams key mappings.\n self.key_sources = self.stream_keys[\"sources\"]\n self.key_targets = self.stream_keys[\"targets\"]\n\n # Get absolute path to data folder.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get dataset.\n if (self.config['dataset'] is None) or (self.config['dataset'] not in [\"eng-fra\", \"eng-pol\"]):\n raise ConfigurationError(\"Task supports only 'dataset' options: 'eng-fra', 'eng-pol'\")\n dataset = self.config['dataset']\n\n # Get (sub)set: train/valid/test.\n if (self.config['subset'] is None) or (self.config['subset'] not in ['train', 'valid', 'test']):\n raise ConfigurationError(\"Task supports one 'subset' options: 'train', 'valid', 'test' \")\n subset = self.config['subset']\n\n # Extract source and target language name\n self.lang_source = self.config['dataset'].split('-')[0]\n self.lang_target = self.config['dataset'].split('-')[1]\n\n\n # Names of files used by this task.\n filenames = [\n self.lang_source + \".train.txt\",\n self.lang_target + \".train.txt\", \n self.lang_source + \".valid.txt\", \n self.lang_target + \".valid.txt\", \n self.lang_source + \".test.txt\", \n self.lang_target + \".test.txt\"\n ]\n\n # Initialize dataset if files do not exist.\n if not io.check_files_existence(os.path.join(self.data_folder, dataset), filenames):\n # Set url and source filename depending on dataset.\n url = \"https://www.manythings.org/anki/\" + self.lang_target + \"-\" + self.lang_source + \".zip\"\n zipfile_name = \"translate_\" + self.lang_target + \"_\" + self.lang_source + \".zip\"\n\n with tempfile.TemporaryDirectory() as tmpdirname:\n # Download and extract wikitext zip.\n io.download_extract_zip_file(self.logger, tmpdirname, url, zipfile_name)\n\n # Create train, valid, test files from the downloaded file\n lines = io.load_string_list_from_txt_file(tmpdirname, self.lang_target + \".txt\")\n\n # Shuffle the lines\n random.seed(42)\n random.shuffle(lines)\n\n # Split english and french pairs\n lines_source = [self.normalizeString(l.split('\\t')[0]) for l in lines]\n lines_target = [self.normalizeString(l.split('\\t')[1]) for l in lines]\n\n # Cut dataset into train (90%), valid (5%), test (5%) files\n test_index = len(lines) // 20\n valid_index = test_index + (len(lines) // 20)\n\n os.makedirs(os.path.join(self.data_folder, dataset), exist_ok=True)\n \n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_source + \".test.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_source[0:test_index]))\n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_target + \".test.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_target[0:test_index]))\n\n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_source + \".valid.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_source[test_index:valid_index]))\n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_target + \".valid.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_target[test_index:valid_index]))\n\n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_source + \".train.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_source[valid_index:]))\n with open(os.path.join(os.path.join(self.data_folder, dataset), self.lang_target + \".train.txt\"), mode='w+') as f:\n f.write('\\n'.join(lines_target[valid_index:]))\n\n else:\n self.logger.info(\"Files {} found in folder '{}'\".format(filenames, self.data_folder))\n\n\n # Load the lines\n lines_source = io.load_string_list_from_txt_file(os.path.join(self.data_folder, dataset), self.lang_source + \".\"+subset+\".txt\")\n lines_target = io.load_string_list_from_txt_file(os.path.join(self.data_folder, dataset), self.lang_target + \".\"+subset+\".txt\")\n\n # Get the required sample length.\n self.sentence_length = self.config['sentence_length']\n\n # Separate into src - tgt sentence pairs + tokenize\n tokenizer = WhitespaceTokenizer()\n self.sentences_source = []\n self.sentences_target = []\n for s_src, s_tgt in zip(lines_source, lines_target):\n src = tokenizer.tokenize(s_src)\n tgt = tokenizer.tokenize(s_tgt)\n # Keep only the pairs that are shorter or equal to the requested length\n # If self.sentence_length < 0, then give all the pairs regardless of length\n if (len(src) <= self.sentence_length and len(tgt) <= self.sentence_length) \\\n or self.sentence_length < 0:\n self.sentences_source += [src]\n self.sentences_target += [tgt]\n\n self.logger.info(\"Load text consisting of {} sentences\".format(len(self.sentences_source)))\n\n # Calculate the size of dataset.\n self.dataset_length = len(self.sentences_source)\n\n # Display exemplary sample.\n self.logger.info(\"Exemplary sample:\\n source: {}\\n target: {}\".format(self.sentences_source[0], self.sentences_target[0]))\n \n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_sources: DataDefinition([-1, self.sentence_length, 1], [list, list, str], \"Batch of input sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]\"),\n self.key_targets: DataDefinition([-1, self.sentence_length, 1], [list, list, str], \"Batch of target sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]\")\n }\n\n # Turn a Unicode string to plain ASCII, thanks to\n # https://stackoverflow.com/a/518232/2809427\n @staticmethod\n def unicodeToAscii(s):\n return ''.join(\n c for c in unicodedata.normalize('NFD', s)\n if unicodedata.category(c) != 'Mn'\n )\n\n # Lowercase, trim, and remove non-letter characters\n # https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n def normalizeString(self, s):\n s = self.unicodeToAscii(s.lower().strip())\n s = re.sub(r\"([.!?])\", r\" \\1\", s)\n s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n return s\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return self.dataset_length\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'indices', sources','targets'})``\n\n \"\"\"\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_sources] = self.sentences_source[index]\n data_streams[self.key_targets] = self.sentences_target[index]\n return data_streams\n\n def collate_fn(self, batch):\n \"\"\"\n Generates a batch of samples from a list of individuals samples retrieved by :py:func:`__getitem__`.\n\n :param batch: List of :py:class:`ptp.utils.DataStreams` retrieved by :py:func:`__getitem__`\n :type batch: list\n\n :return: DataStreams containing the created batch.\n\n \"\"\"\n # Collate indices.\n data_streams = self.create_data_streams([sample[self.key_indices] for sample in batch])\n # Collate sources.\n data_streams[self.key_sources] = [sample[self.key_sources] for sample in batch]\n data_streams[self.key_targets] = [sample[self.key_targets] for sample in batch]\n return data_streams\n\n" }, { "path": "ptp/components/tasks/text_to_text/wikitext_language_modeling.py", "content": "# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport os\n\nfrom nltk.tokenize import WhitespaceTokenizer\n\nimport ptp.components.mixins.io as io\nfrom ptp.configuration import ConfigurationError\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass WikiTextLanguageModeling(Task):\n \"\"\"\n Language modeling task using WikiText-2 (_dataset2) / WikiText-103 (_dataset103) datasets, featured at the Salesforce _website.\n\n Task downloads the files, loads the file associated with a given subset (train/valid/test), concatenates all sencentes and tokenizes them using NLTK's WhitespaceTokenizer.\n \n Resulting tokens are then passed to samples (source/target) as list of tokens of a given length (set by the user in configuration file).\n\n Associated paper: Stephen Merity, Caiming Xiong, James Bradbury, and Richard Socher. Pointer Sentinel Mixture Models (2016) (_arxiv)\n\n .. _website: https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/\n .. _dataset2: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip\n .. _dataset103: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip\n .. _arxiv: https://arxiv.org/abs/1609.07843\n \"\"\"\n def __init__(self, name, config):\n \"\"\"\n The init method downloads the required files, loads the file associated with a given subset (train/valid/test), \n concatenates all sencentes and tokenizes them using NLTK's WhitespaceTokenizer.\n\n It also stores the intermediate results, so for example, it file with tokenized set is found, it simply loads it.\n\n :param name: Name of the component.\n\n :param class_type: Class type of the component.\n\n :param config: Dictionary of parameters (read from configuration ``.yaml`` file).\n \"\"\"\n # Call constructor of parent classes.\n Task.__init__(self, name, WikiTextLanguageModeling, config) \n\n # Set streams key mappings.\n self.key_sources = self.stream_keys[\"sources\"]\n self.key_targets = self.stream_keys[\"targets\"]\n\n # Get absolute path to data folder.\n self.data_folder = os.path.expanduser(self.config['data_folder'])\n\n # Get dataset.\n if (self.config['dataset'] is None) or (self.config['dataset'] not in [\"wikitext-2\", \"wikitext-103\"]):\n raise ConfigurationError(\"Task supports two 'dataset' options: 'wikitext-2', 'wikitext-103' \")\n dataset = self.config['dataset']\n\n # Get (sub)set: train/valid/test.\n if (self.config['subset'] is None) or (self.config['subset'] not in ['train', 'valid', 'test']):\n raise ConfigurationError(\"Task supports three 'subset' options: 'train', 'valid', 'test' \")\n subset = self.config['subset']\n\n # Check if file with tokenized words exists.\n filename_tokenized_words = \"wiki.\"+self.config['subset']+\".tokenized_words\"\n\n if not io.check_files_existence(self.data_folder, filename_tokenized_words):\n # If not, we must generate (and save it) using source files.\n\n # Names of files used by this task.\n filenames = [\"wiki.train.tokens\", \"wiki.valid.tokens\", \"wiki.test.tokens\"]\n\n # Initialize dataset if files do not exist.\n if not io.check_files_existence(self.data_folder, filenames):\n # Set url and source filename depending on dataset.\n if dataset == \"wikitext-2\":\n url = \"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip\"\n zipfile_name = \"wikitext-2-v1.zip\"\n else: \n url = \"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip\"\n zipfile_name = \"wikitext-103-v1.zip\"\n\n # Download and extract wikitext zip.\n io.download_extract_zip_file(self.logger, self.data_folder, url, zipfile_name)\n\n # Move extracted files to the right folder.\n io.move_files_between_dirs(self.logger, os.path.join(self.data_folder, dataset) , self.data_folder, filenames)\n else:\n self.logger.info(\"Files {} found in folder '{}'\".format(filenames, self.data_folder))\n\n\n # Load the whole sentences.\n sentences = io.load_string_list_from_txt_file(self.data_folder, \"wiki.\"+subset+\".tokens\")\n self.logger.info(\"Loaded {} sentences from the 'wiki.{}.tokens' subset\".format(len(sentences), subset))\n\n # Generate text full of tokens.\n self.logger.info(\"Please wait, using NLTK to tokenize the loaded sentences...\")\n # Create a single text by replacing newlines with tokens.\n text = \" \".join(sentences)\n # Tokenize.\n tokenizer = WhitespaceTokenizer()\n self.tokens = tokenizer.tokenize(text)\n # Save fo file.\n io.save_string_list_to_txt_file(self.data_folder, filename_tokenized_words, self.tokens)\n self.logger.info(\"Created text consisting of {} tokens and saved it to '{}'\".format(len(self.tokens), filename_tokenized_words))\n else:\n # Ok, file with tokens exists, load it.\n self.tokens = io.load_string_list_from_txt_file(self.data_folder, filename_tokenized_words)\n self.logger.info(\"Load text consisting of {} tokens from '{}'\".format(len(self.tokens), filename_tokenized_words))\n\n # Get the required sample length.\n self.sentence_length = self.config['sentence_length']\n # Calculate the size of dataset.\n self.dataset_length = len(self.tokens) - self.sentence_length - 1 # as target is \"shifted\" by 1.\n\n # Display exemplary sample.\n self.logger.info(\"Exemplary sample:\\n source: {}\\n target: {}\".format(self.tokens[0:self.sentence_length], self.tokens[1:self.sentence_length+1]))\n \n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_sources: DataDefinition([-1, self.sentence_length, 1], [list, list, str], \"Batch of input sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]\"),\n self.key_targets: DataDefinition([-1, self.sentence_length, 1], [list, list, str], \"Batch of target sentences, each consisting of several words [BATCH_SIZE] x [SENTENCE_LENGTH] x [string]\")\n }\n\n\n def __len__(self):\n \"\"\"\n Returns the \"size\" of the \"task\" (total number of samples).\n\n :return: The size of the task.\n \"\"\"\n return self.dataset_length\n\n\n def __getitem__(self, index):\n \"\"\"\n Getter method to access the dataset and return a sample.\n\n :param index: index of the sample to return.\n :type index: int\n\n :return: ``DataStreams({'indices', sources','targets'})``\n\n \"\"\"\n # Return data_streams.\n data_streams = self.create_data_streams(index)\n data_streams[self.key_sources] = self.tokens[index:index+self.sentence_length]\n data_streams[self.key_targets] = self.tokens[index+1:index+self.sentence_length+1] # target is \"shifted\" by 1.\n #print(\"task: index = {} source = {} target = {}\".format(index, data_streams[self.key_sources], data_streams[self.key_targets]))\n return data_streams\n\n def collate_fn(self, batch):\n \"\"\"\n Generates a batch of samples from a list of individuals samples retrieved by :py:func:`__getitem__`.\n\n :param batch: List of :py:class:`ptp.utils.DataStreams` retrieved by :py:func:`__getitem__`\n :type batch: list\n\n :return: DataStreams containing the created batch.\n\n \"\"\"\n # Collate indices.\n data_streams = self.create_data_streams([sample[self.key_indices] for sample in batch])\n # Collate sources.\n data_streams[self.key_sources] = [sample[self.key_sources] for sample in batch]\n data_streams[self.key_targets] = [sample[self.key_targets] for sample in batch]\n return data_streams\n\n" }, { "path": "ptp/components/transforms/__init__.py", "content": "from .concatenate_tensor import ConcatenateTensor\nfrom .list_to_tensor import ListToTensor\nfrom .reduce_tensor import ReduceTensor\nfrom .reshape_tensor import ReshapeTensor\n\n\n__all__ = [\n 'ConcatenateTensor',\n 'ListToTensor',\n 'ReduceTensor',\n 'ReshapeTensor',\n ]\n" }, { "path": "ptp/components/transforms/concatenate_tensor.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass ConcatenateTensor(Component):\n \"\"\"\n Class responsible for concatenation of list of input tensors into a single tensor.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, ConcatenateTensor, config)\n\n # Get key mappings.\n self.key_outputs = self.stream_keys[\"outputs\"]\n\n # Load list of streams names (keys).\n self.input_stream_keys = self.config[\"input_streams\"]\n if type(self.input_stream_keys) == str:\n self.input_stream_keys = self.input_stream_keys.replace(\" \", \"\").split(\",\")\n \n\n # Get input shapes from configuration.\n # Assuming that it will be list of lists.\n self.input_stream_dims = [[int(x) for x in dims] for dims in self.config[\"input_dims\"]]\n\n # Get output shape from configuration.\n self.output_dims = [int(x) for x in self.config[\"output_dims\"]]\n\n # Get concatenation dimension.\n self.dim = self.config[\"dim\"]\n\n # Set global variable - all dimensions ASIDE OF BATCH.\n self.globals[\"output_size\"] = self.output_dims[1:]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n keys: DataDefinition(dims, [torch.Tensor], \"Batch of inputs [BATCH_SIZE x ...]\")\n for (keys, dims) in zip(self.input_stream_keys, self.input_stream_dims)\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {\n self.key_outputs: DataDefinition(self.output_dims, [torch.Tensor], \"Batch of outputs [BATCH_SIZE x ... ]\"),\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs to be concatentated.\n inputs = [data_streams[stream_key] for stream_key in self.input_stream_keys]\n\n #print(\"{}: input shape: {}, device: {}\\n\".format(self.name, [input.shape for input in inputs], [input.device for input in inputs]))\n\n\n # Concat.\n outputs = torch.cat(inputs, dim=self.dim)\n\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs})\n\n" }, { "path": "ptp/components/transforms/list_to_tensor.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass ListToTensor(Component):\n \"\"\"\n Class responsible for transforming list (of lists) to a tensor.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Name of the component loaded from the configuration file.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, ListToTensor, config)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n \n # Get number of input dimensions from configuration.\n self.num_inputs_dims = self.config[\"num_inputs_dims\"]\n\n # Get size of a single input item (last dimension) from globals.\n self.input_size = self.globals[\"input_size\"]\n\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition(\n [-1]*(self.num_inputs_dims-1) + [self.input_size],\n [list]*(self.num_inputs_dims-1) + [torch.Tensor],\n \"Batch of inputs [DIM 1 x DIM 2 x ... x INPUT_SIZE]\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {\n self.key_outputs: DataDefinition(\n [-1]*(self.num_inputs_dims-1) + [self.input_size],\n [torch.Tensor],\n \"Batch of outputs [DIM 1 x DIM 2 x ... x INPUT_SIZE]\")\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n\n # Change to tensor.\n if self.num_inputs_dims == 1:\n # CASE: Single tensor.\n outputs = input\n elif self.num_inputs_dims == 2:\n # CASE: List of tensors.\n\n # This needs testing - padding?\n outputs = torch.stack(inputs)\n\n elif self.num_inputs_dims == 3:\n # CASE: List of lists of tensors.\n\n # Get type.\n ttype = type(inputs[0][0])\n # Generate tensor that will be added as padding - all zeros.\n pad_tensor = ttype([0]*self.input_size)\n if self.app_state.use_gpu:\n pad_tensor = pad_tensor.cuda()\n\n # Get max length of lists.\n max_len = max([len(lst) for lst in inputs])\n\n # List of stacked tensors.\n stacked_tensor_lst = []\n\n # Iterate over list of lists.\n for lst in inputs:\n # \"Manual\" padding of each inner list.\n if len(lst) < max_len:\n lst.extend([pad_tensor] * (max_len - len(lst)))\n # Stack inner list.\n stacked_tensor = torch.stack(lst)\n stacked_tensor_lst.append(stacked_tensor)\n # Finally, pad the result.\n outputs = torch.stack(stacked_tensor_lst)\n\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs})\n\n" }, { "path": "ptp/components/transforms/reduce_tensor.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.configuration.config_parsing import get_value_from_dictionary\n\n\nclass ReduceTensor(Component):\n \"\"\"\n Class responsible for reducing tensor using indicated reduction method along a given dimension.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Name of the component loaded from the configuration file.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, ReduceTensor, config)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n \n # Get number of input dimensions from configuration.\n self.num_inputs_dims = self.config[\"num_inputs_dims\"]\n # Get size of a single input item (last dimension) from globals.\n self.input_size = self.globals[\"input_size\"]\n\n # Get reduction tparamsype from configuration.\n self.dim = self.config[\"reduction_dim\"]\n self.keepdim = self.config[\"keepdim\"]\n\n # Set reduction type.\n rt = get_value_from_dictionary(\n \"reduction_type\", self.config,\n 'sum | mean | min | max | argmin | argmax'.split(\" | \")\n )\n reduction_types = {}\n reduction_types[\"sum\"] = torch.sum\n reduction_types[\"mean\"] = torch.mean\n reduction_types[\"min\"] = torch.min\n reduction_types[\"max\"] = torch.max\n reduction_types[\"argmin\"] = torch.argmin\n reduction_types[\"argmax\"] = torch.argmax\n\n self.reduction = reduction_types[rt]\n\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n # Generate the description of input stream.\n dims_desc = [\"DIM {}\".format(i) for i in range(self.num_inputs_dims-1)]\n desc = \"Batch of outputs [\" + \" x \".join(dims_desc) + \"]\"\n return {\n self.key_inputs: DataDefinition(\n [-1]*(self.num_inputs_dims-1) + [self.input_size],\n [torch.Tensor],\n desc)\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n # Generate the dimensions and description of output stream.\n if self.keepdim:\n dims = [-1]*(self.num_inputs_dims-1) + [self.input_size]\n dims[self.dim] = 1\n dims_desc = [\"DIM {}\".format(i) for i in range(self.num_inputs_dims)]\n dims_desc[self.dim] = \"1\"\n desc = \"Batch of outputs [\" + \" x \".join(dims_desc) + \"]\"\n else:\n dims = [-1]*(self.num_inputs_dims-2) + [self.input_size]\n dims_desc = [\"DIM {}\".format(i) for i in range(self.num_inputs_dims-1)]\n desc = \"Batch of outputs [\" + \" x \".join(dims_desc) + \"]\"\n return {\n self.key_outputs: DataDefinition(\n dims,\n [torch.Tensor],\n desc)\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n\n outputs = self.reduction(inputs, self.dim, self.keepdim)\n\n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs})\n\n" }, { "path": "ptp/components/transforms/reshape_tensor.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport torch\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass ReshapeTensor(Component):\n \"\"\"\n Class responsible for reshaping the input tensor.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, ReshapeTensor, config)\n\n # Set key mappings.\n self.key_inputs = self.stream_keys[\"inputs\"]\n self.key_outputs = self.stream_keys[\"outputs\"]\n \n # Get input and output shapes from configuration.\n self.input_dims = [int(x) for x in self.config[\"input_dims\"]]\n self.output_dims = [int(x) for x in self.config[\"output_dims\"]]\n\n # Set global variable - all dimensions ASIDE OF BATCH.\n self.globals[\"output_size\"] = self.output_dims[1:]\n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.utils.DataDefinition`).\n \"\"\"\n return {\n self.key_inputs: DataDefinition(self.input_dims, [torch.Tensor], \"Batch of inputs [BATCH_SIZE x ...]\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a empty dictionary with definitions of output data produced the component.\n\n :return: Empty dictionary.\n \"\"\"\n return {\n self.key_outputs: DataDefinition(self.output_dims, [torch.Tensor], \"Batch of outputs [BATCH_SIZE x ... ]\"),\n }\n\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes \"inputs\" in the format of a single tensor.\n Stores reshaped tensor in \"outputs\" field of in data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others):\n\n - \"inputs\": expected input field containing tensor [BATCH_SIZE x ...]\n\n - \"outputs\": added output field containing tensor [BATCH_SIZE x ...] \n \"\"\"\n # Get inputs to be encoded.\n inputs = data_streams[self.key_inputs]\n\n # Reshape.\n outputs = inputs.view(self.output_dims) \n # Create the returned dict.\n data_streams.publish({self.key_outputs: outputs})\n\n" }, { "path": "ptp/components/viewers/__init__.py", "content": "from .image_viewer import ImageViewer\nfrom .stream_viewer import StreamViewer\n\n__all__ = [\n 'ImageViewer',\n 'StreamViewer',\n ]\n" }, { "path": "ptp/components/viewers/image_viewer.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport numpy as np\nimport torch\nimport matplotlib.pyplot as plt\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass ImageViewer(Component):\n \"\"\"\n Utility for displaying contents image along with label and prediction (a single sample from the batch).\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes loss object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, ImageViewer, config)\n\n # Get default key mappings.\n self.key_indices = self.stream_keys[\"indices\"]\n self.key_images = self.stream_keys[\"images\"]\n self.key_labels = self.stream_keys[\"labels\"]\n self.key_answers = self.stream_keys[\"answers\"]\n\n # Get sample number.\n self.sample_number = self.config[\"sample_number\"]\n \n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n self.key_images: DataDefinition([-1, -1, -1, -1], [torch.Tensor], \"Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]\"),\n self.key_labels: DataDefinition([-1, 1], [list, str], \"Batch of target labels, each being a single word [BATCH_SIZE] x [STRING]\"),\n self.key_answers: DataDefinition([-1, 1], [list, str], \"Batch of predicted labels, each being a single word [BATCH_SIZE] x [STRING]\")\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Shows a sample from the batch.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object.\n\n \"\"\"\n # Use worker interval.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n\n # Get inputs\n indices = data_streams[self.key_indices]\n images = data_streams[self.key_images]\n labels = data_streams[self.key_labels]\n answers = data_streams[self.key_answers]\n\n # Get sample number.\n if self.sample_number == -1:\n # Random.\n sample_number = np.random.randint(0, len(images))\n else:\n sample_number = self.sample_number\n\n # Get \"sample\".\n image = images[sample_number].cpu().data.numpy()\n label = labels[sample_number]\n answer = answers[sample_number]\n\n # Reshape image.\n if image.shape[0] == 1:\n # This is a single channel image - get rid of this dimension\n image = np.squeeze(image, axis=0)\n else:\n # More channels - move channels to axis2, according to matplotilb documentation.\n # (X : array_like, shape (n, m) or (n, m, 3) or (n, m, 4))\n image = image.transpose(1, 2, 0)\n\n # Show data.\n plt.title('Sample: {} (index: {})\\nPrediction: {} | Target: {}'.format(sample_number, indices[sample_number], answer, label))\n plt.imshow(image, interpolation='nearest', aspect='auto')\n\n # Plot!\n plt.show()\n\n\n" }, { "path": "ptp/components/viewers/stream_viewer.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport numpy as np\n\nfrom ptp.configuration.config_parsing import get_value_list_from_dictionary\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\n\n\nclass StreamViewer(Component):\n \"\"\"\n Utility for displaying contents of streams of a single sample from the batch.\n\n \"\"\"\n\n def __init__(self, name, config):\n \"\"\"\n Initializes loss object.\n\n :param name: Loss name.\n :type name: str\n\n :param config: Dictionary of parameters (read from the configuration ``.yaml`` file).\n :type config: :py:class:`ptp.configuration.ConfigInterface`\n\n \"\"\"\n # Call constructors of parent classes.\n Component.__init__(self, name, StreamViewer, config)\n\n # Get key mappings for indices.\n self.key_indices = self.stream_keys[\"indices\"]\n\n # Load list of streams names (keys).\n self.input_stream_keys = get_value_list_from_dictionary(\"input_streams\", self.config)\n \n # Get sample number.\n self.sample_number = self.config[\"sample_number\"]\n \n\n def input_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of input data that are required by the component.\n\n :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n self.key_indices: DataDefinition([-1, 1], [list, int], \"Batch of sample indices [BATCH_SIZE] x [1]\"),\n }\n\n def output_data_definitions(self):\n \"\"\" \n Function returns a dictionary with definitions of output data produced the component.\n\n :return: dictionary containing output data definitions (each of type :py:class:`ptp.data_types.DataDefinition`).\n \"\"\"\n return {\n }\n\n def __call__(self, data_streams):\n \"\"\"\n Encodes batch, or, in fact, only one field of batch (\"inputs\").\n Stores result in \"outputs\" field of data_streams.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object containing (among others) \"indices\".\n\n \"\"\"\n # Use worker interval.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n\n # Get indices.\n indices = data_streams[self.key_indices]\n\n # Get sample number.\n if self.sample_number == -1:\n # Random\n sample_number = np.random.randint(0, len(indices))\n else:\n sample_number = self.sample_number\n\n # Generate displayed string.\n absent_streams = []\n disp_str = \"Showing selected streams for sample {} (index: {}):\\n\".format(sample_number, indices[sample_number])\n for stream_key in self.input_stream_keys:\n if stream_key in data_streams.keys():\n disp_str += \" '{}': {}\\n\".format(stream_key, data_streams[stream_key][sample_number])\n else:\n absent_streams.append(stream_key)\n\n # Log values and inform about missing streams.\n self.logger.info(disp_str)\n if len(absent_streams) > 0:\n self.logger.warning(\"Could not display the following (absent) streams: {}\".format(absent_streams))\n" }, { "path": "ptp/configuration/__init__.py", "content": "from .config_interface import ConfigInterface\nfrom .config_registry import ConfigRegistry\nfrom .configuration_error import ConfigurationError\n#from configs_parsing import load_default_configuration_file\n\n__all__ = [\n 'ConfigInterface',\n 'ConfigRegistry',\n 'ConfigurationError',\n ]\n" }, { "path": "ptp/configuration/config_interface.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman, Tomasz Kornuta\"\n\nimport yaml\nfrom collections.abc import Mapping\n\nfrom ptp.configuration.config_registry import ConfigRegistry\n\n\nclass ConfigInterface(Mapping):\n \"\"\"\n Interface to the :py:class:`ConfigRegistry` singleton.\n\n Inherits :py:class:`collections.Mapping`, and therefore exposes functionality close to a `dict`.\n\n Offers a read (through :py:class:`collections.Mapping` interface) and write \\\n (through :py:func:`add_default_params` and :py:func:`add_config_params` methods) \\\n view of the :py:class:`ConfigRegistry`.\n\n .. warning::\n\n This class is the only interface to :py:class:`ConfigRegistry`, and thus the only way to \\\n interact with it.\n\n \"\"\"\n\n def __init__(self, *keys):\n \"\"\"\n Constructor:\n\n - Call base constructor (:py:class:`Mapping`),\n - Initializes the :py:class:`ConfigRegistry`,\n - Initializes empty keys_path list\n\n\n :param keys: Sequence of keys to the subtree of the registry. The subtree hierarchy will be created if it \\\n does not exist. If empty, shows the whole registry.\n :type keys: sequence / collection: dict, list etc.\n\n .. note::\n\n Calling :py:func:`to_dict` after initializing a :py:class:`ConfigInterface` with ``keys``, \\\n will throw a ``KeyError``.\n\n Adding `default` & `config` params should be done through :py:func:`add_default_param` and \\\n :py:func:`add_config_param`.\n\n ``keys`` is mainly purposed for the recursion of :py:class:`ConfigInterface`.\n\n \"\"\"\n # call base constructor\n super(ConfigInterface, self).__init__()\n\n # empty ConfigRegistry\n self._config_registry = ConfigRegistry()\n\n # keys_path as a list\n self._keys_path = list(keys)\n\n def _lookup(self, *keys):\n \"\"\"\n Returns the :py:class:`ConfigInterface` or the value living under ``keys``.\n\n :param keys: Sequence of keys to the subtree of the registry. If empty, shows the whole registry.\n :type keys: sequence / collection: dict, list etc.\n\n \"\"\"\n\n def lookup_recursion(dic, key, *keys):\n if keys:\n return lookup_recursion(dic[key], *keys)\n return dic[key]\n\n # construct the path from the existing keys path\n lookup_keys = self._keys_path + list(keys)\n\n if len(lookup_keys) > 0:\n r = lookup_recursion(self._config_registry, *lookup_keys)\n return r\n else:\n return self._config_registry\n\n def _nest_dict(self, d: dict):\n \"\"\"\n Create a nested dict using ``d`` living under ``self._keys_path``.\n\n :param d: dict to nest under ``self._keys_path``.\n :type d: dict\n\n :return: nested ``d``.\n\n \"\"\"\n\n def nest_dict_recursion(dic, key, *keys):\n if keys:\n dic[key] = {}\n return nest_dict_recursion(dic[key], *keys)\n else:\n dic[key] = {}\n dic[key].update(d)\n\n if len(self._keys_path) > 0:\n nested_dict = {}\n nest_dict_recursion(nested_dict, *self._keys_path)\n return nested_dict\n else:\n return d\n\n def to_dict(self):\n \"\"\"\n\n :return: `dict` containing a snapshot of the current :py:class:`ConfigInterface` tree.\n \"\"\"\n return dict(self._lookup())\n\n def __getitem__(self, key):\n \"\"\"\n Get parameter value under ``key``.\n\n The parameter dict is derived from the default parameters updated with the config parameters.\n\n :param key: key to value in the :py:class:`ConfigInterface` tree.\n :type key: str\n\n :return: :py:class:`ConfigInterface` ``[key]`` or value if leaf of the :py:class:`ConfigRegistry` tree.\n\n \"\"\"\n v = self._lookup(key)\n if isinstance(v, dict) or isinstance(v, ConfigRegistry):\n return ConfigInterface(*self._keys_path, key)\n else: # We are at a leaf of the tree\n return v\n\n def __len__(self):\n \"\"\"\n\n :return: Length of the :py:class:`ConfigInterface`.\n\n \"\"\"\n return len(self._lookup())\n\n def __iter__(self):\n \"\"\"\n\n :return: Iterator over the :py:class:`ConfigInterface`.\n\n \"\"\"\n return iter(self._lookup())\n\n def __eq__(self, other):\n \"\"\"\n Check whether two interfaces are equal (just for the purpose of compatibility with base Mapping class).\n \"\"\"\n if isinstance(other, self.__class__):\n\n return self._config_registry == other._config_registry and self._keys_path == other._keys_path\n else:\n return False\n\n \n def leafs(self):\n \"\"\"\n Yields the leafs of the current :py:class:`ConfigInterface`.\n\n \"\"\"\n for key, value in self.items():\n if isinstance(value, ConfigInterface):\n for inner_key in value.leafs():\n yield inner_key\n else:\n yield key\n\n def set_leaf(self, leaf_key, leaf_value):\n \"\"\"\n Update the value of the specified ``leaf_key`` of the current :py:class:`ConfigInterface` \\\n with the specified ``leaf_value``.\n\n :param leaf_key: leaf key to update.\n :type leaf_key: str\n\n :param leaf_value: New value to set.\n\n :return: ``True`` if the leaf value has been changed, ``False`` if ``leaf_key`` is not in \\\n :py:func:`ConfigInterface.leafs`.\n\n \"\"\"\n # check first if we can access the leaf to change\n if leaf_key not in list(self.leafs()):\n return False\n\n for key, value in self.items():\n\n if isinstance(value, ConfigInterface):\n # hit a sub ConfigInterface, recursion\n if value.set_leaf(leaf_key, leaf_value):\n return True # leaf has been changed, done\n else:\n continue # have not found the key, continue\n elif key == leaf_key:\n self.add_config_params({key: leaf_value})\n return True # leaf has been changed, done\n\n def add_default_params(self, default_params: dict):\n \"\"\"\n Appends ``default_params`` to the `config` parameter dict of the :py:class:`ConfigRegistry`.\n\n .. note::\n\n This method should be used by the objects necessitating default values \\\n (tasks, models, workers etc.).\n\n :param default_params: Dictionary containing `default` values.\n :type default_params: dict\n\n The dictionary will be inserted into the subtree keys path indicated at the initialization of the \\\n current :py:class:`ConfigInterface`.\n\n \"\"\"\n self._config_registry.add_default_params(\n self._nest_dict(default_params)\n )\n\n def add_config_params(self, config_params: dict):\n \"\"\"\n Appends ``config_params`` to the `config` parameter dict of the :py:class:`ConfigRegistry`.\n\n .. note::\n\n This is intended for the user to dynamically (re)configure his experiments.\n\n :param config_params: Dictionary containing `config` values.\n :type config_params: dict\n\n The dictionary will be inserted into the subtree keys path indicated at the initialization of the \\\n current :py:class:`ConfigInterface`.\n\n \"\"\"\n self._config_registry.add_config_params(\n self._nest_dict(config_params)\n )\n\n def del_default_params(self, key):\n \"\"\"\n Removes the entry from the `default` params living under ``key``.\n\n The entry can either be a subtree or a leaf of the `default` params tree.\n\n :param key: key to subtree / leaf in the `default` params tree.\n :type key: str\n\n \"\"\"\n if type(key) is list:\n keypath = self._keys_path + key\n else:\n keypath = self._keys_path + [key]\n self._config_registry.del_default_params(keypath)\n\n def del_config_params(self, key):\n \"\"\"\n Removes the entry from the `config` params living under ``key``.\n\n The entry can either be a subtree or a leaf of the `config` params tree.\n\n :param key: key to subtree / leaf in the `config` params tree.\n :type key: str\n\n \"\"\"\n if type(key) is list:\n keypath = self._keys_path + key\n else:\n keypath = self._keys_path + [key]\n self._config_registry.del_config_params(keypath)\n\n def add_config_params_from_yaml(self, yaml_path: str):\n \"\"\"\n Helper function adding `config` params by loading the file at ``yaml_path``.\n\n Wraps call to :py:func:`add_default_param`.\n\n :param yaml_path: Path to a ``.yaml`` file containing config parameters.\n :type yaml_path: str`\n\n \"\"\"\n # Open file and try to add that to list of parameter dictionaries.\n with open(yaml_path, 'r') as stream:\n # Load parameters.\n params_from_yaml = yaml.safe_load(stream)\n\n # add config param\n self.add_config_params(params_from_yaml)\n" }, { "path": "ptp/configuration/config_parsing.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport yaml\n\nfrom ptp.utils.app_state import AppState\nfrom ptp.configuration.configuration_error import ConfigurationError\n\n\ndef display_globals(logger, globals_dict):\n \"\"\"\n Displays the global variables.\n\n :param logger: logger object\n\n :param globals_dict: Dictionary with globals\n \"\"\"\n # Create the string.\n global_str = 'Final global variables:\\n'\n global_str += '='*80 + '\\n'\n for key,value in globals_dict:\n global_str += \" {}: {}\\n\".format(key, value)\n global_str += '='*80 + '\\n'\n # Display.\n logger.info(global_str)\n\n\ndef display_parsing_results(logger, parsed_args, unparsed_args):\n \"\"\"\n Displays the properly & improperly parsed arguments (if any).\n\n :param logger: logger object\n\n :param parsed_args: Parsed command-line arguments\n\n :param unparsed_args: Unparsed command-line arguments\n\n \"\"\"\n # Log the parsed flags.\n flags_str = 'Properly parsed command line arguments: \\n'\n flags_str += '='*80 + '\\n'\n for arg in vars(parsed_args): \n flags_str += \" {}= {} \\n\".format(arg, getattr(parsed_args, arg))\n flags_str += '='*80 + '\\n'\n logger.info(flags_str)\n\n # Log the unparsed flags if any.\n if unparsed_args:\n flags_str = 'Invalid command line arguments: \\n'\n flags_str += '='*80 + '\\n'\n for arg in unparsed_args: \n flags_str += \" {} \\n\".format(arg)\n flags_str += '='*80 + '\\n'\n logger.warning(flags_str)\n\n\ndef export_experiment_configuration_to_yml(logger, log_dir, filename, config_interface_obj, user_confirm):\n \"\"\"\n Dumps the configuration to ``yaml`` file.\n\n :param logger: logger object\n\n :param log_dir: Directory used to host log files (such as the collected statistics).\n :type log_dir: str\n\n :param filename: Name of the ``yaml`` file to write to.\n :type filename: str\n\n :param config_interface_obj: Configuration interface object.\n\n :param user_confirm: Whether to request user confirmation.\n :type user_confirm: bool\n\n \"\"\"\n # -> At this point, all configuration for experiment is complete.\n\n # Log the resulting training configuration.\n conf_str = 'Final parameter registry configuration:\\n'\n conf_str += '='*80 + '\\n'\n conf_str += yaml.safe_dump(config_interface_obj.to_dict(), default_flow_style=False)\n conf_str += '='*80 + '\\n'\n \n logger.info(conf_str)\n\n # Save the resulting configuration into a .yaml settings file, under log_dir\n with open(log_dir + filename, 'w') as yaml_backup_file:\n yaml.dump(config_interface_obj.to_dict(), yaml_backup_file, default_flow_style=False)\n\n # Ask for confirmation - optional.\n if user_confirm:\n try:\n input('Press to confirm and start the experiment\\n')\n except KeyboardInterrupt:\n exit(0) \n\n\ndef load_class_default_config_file(class_type):\n \"\"\"\n Function loads default configuration from the default config file associated with the given class type and adds it to parameter registry.\n\n :param class_type: Class type of a given object.\n\n :raturn: Loaded default configuration.\n \"\"\"\n \n # Extract path to default config.\n module = class_type.__module__.replace(\".\",\"/\")\n rel_path = module[module.find(\"ptp\")+4:]\n # Build the abs path to the default config file of a given component/worker.\n abs_default_config = os.path.join(AppState().absolute_config_path, \"default\", rel_path) + \".yml\"\n\n # Check if file exists.\n if not os.path.isfile(abs_default_config):\n print(\"ERROR: The default configuration file '{}' for '{}' does not exist\".format(abs_default_config, class_type.__module__))\n exit(-1)\n\n try:\n # Open file and get parameter dictionary.\n with open(abs_default_config, 'r') as stream:\n param_dict = yaml.safe_load(stream)\n\n # Return default parameters so they can be added to the global registry.\n if param_dict is None:\n print(\"WARNING: The default configuration file '{}' is empty!\".format(abs_default_config))\n return {}\n else:\n return param_dict\n\n except yaml.YAMLError as e:\n print(\"ERROR: Couldn't properly parse the '{}' default configuration file. YAML error:\\n {}\".format(abs_default_config, e))\n exit(-2)\n\n\ndef recurrent_config_parse(configs_to_parse: list, configs_parsed: list, abs_config_path: str):\n \"\"\"\n Parses names of configuration files in a recursive manner, i.e. \\\n by looking for ``default_config`` sections and trying to load and parse those \\\n files one by one.\n\n :param configs_to_parse: List containing names of configuration files (with paths).\n :type configs_to_parse: list\n\n :param configs_parsed: Configurations that were already parsed (so we won't parse them many times).\n :type configs_parsed: list\n\n :param abs_config_path: Absolute path to ``config`` directory.\n\n :return: list of parsed configuration files.\n\n \"\"\"\n # Terminal condition.\n while len(configs_to_parse) > 0:\n\n # Get config.\n config = configs_to_parse.pop(0)\n\n # Skip empty names (after lose comas).\n if config == '':\n continue\n print(\"Info: Parsing the {} configuration file\".format(config))\n\n # Check if it was already loaded.\n if config in configs_parsed:\n print('Warning: Configuration file {} already parsed - skipping'.format(config))\n continue\n\n # Check if file exists.\n if not os.path.isfile(config):\n print('Error: Configuration file {} does not exist'.format(config))\n exit(-1)\n\n try:\n # Open file and get parameter dictionary.\n with open(config, 'r') as stream:\n param_dict = yaml.safe_load(stream)\n except yaml.YAMLError as e:\n print(\"Error: Couldn't properly parse the {} configuration file\".format(config))\n print('yaml.YAMLERROR:', e)\n exit(-1)\n\n # Remember that we loaded that config.\n configs_parsed.append(config)\n\n # Check if there are any default configs to load.\n if 'default_configs' in param_dict:\n default_configs_to_parse = param_dict['default_configs'].replace(\" \", \"\").split(',')\n # If there are - expand them to absolute paths.\n abs_default_configs_to_parse = [os.path.join(abs_config_path,config) for config in default_configs_to_parse]\n # Recursion!\n configs_parsed = recurrent_config_parse(abs_default_configs_to_parse, configs_parsed, abs_config_path)\n\n # Done, return list of loaded configs.\n return configs_parsed\n\n\ndef reverse_order_config_load(config_interface_obj, configs_to_load):\n \"\"\"\n Loads configuration files in reversed order.\n\n :param config_interface_obj: Configuration interface object.\n\n :param configs_to_load: list of configuration files to load (with absolute paths)\n \"\"\"\n for config in reversed(configs_to_load):\n # Load config from YAML file.\n config_interface_obj.add_config_params_from_yaml(config)\n print('Info: Loaded configuration from file {}'.format(config))\n\n\ndef get_value_list_from_dictionary(key, parameter_dict, accepted_values = []):\n \"\"\"\n Parses parameter values retrieved from a given parameter dictionary using key.\n Optionally, checks is all values are accepted.\n\n :param key: Key of the parameter.\n :param parameter_dict: Dictionary containing given key (e.g. config or globals)\n :param accepted_values: List of accepted values (DEFAULT: [])\n\n :return: List of parsed values\n \"\"\"\n parameter = parameter_dict[key]\n # Preprocess parameter value.\n if (type(parameter) == str):\n if parameter == '':\n # Return empty list.\n return []\n else:\n # Process and split.\n values = parameter.replace(\" \",\"\").split(\",\")\n else:\n values = parameter # list\n assert type(values) == list, \"Parameter value must be a list\"\n\n # Test values one by one.\n if len(accepted_values) > 0:\n for value in values:\n if value not in accepted_values:\n raise ConfigurationError(\"One of the values in '{}' is invalid (current: '{}', accepted: {})\".format(key, value, accepted_values))\n\n # Return list.\n return values\n\ndef get_value_from_dictionary(key, parameter_dict, accepted_values = []):\n \"\"\"\n Parses value of the parameter retrieved from a given parameter dictionary using key.\n Optionally, checks is the values is one of the accepted values.\n\n :param key: Key of the parameter.\n :param parameter_dict: Dictionary containing given key (e.g. config or globals)\n :param accepted_values: List of accepted values (DEFAULT: [])\n\n :return: List of parsed values\n \"\"\"\n value = parameter_dict[key]\n assert type(value) == str, \"Parameter value must be a string\"\n # Preprocess parameter value.\n if value == '':\n return None\n\n # Test values one by one.\n if len(accepted_values) > 0:\n if value not in accepted_values:\n raise ConfigurationError(\"One of the values in '{}' is invalid (current: '{}', accepted: {})\".format(key, value, accepted_values))\n\n # Return value.\n return value\n" }, { "path": "ptp/configuration/config_registry.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman, Tomasz Kornuta\"\n\nimport copy\nfrom abc import ABCMeta\nfrom collections.abc import Mapping\n\nfrom ptp.utils.singleton import SingletonMetaClass\n\n\nclass MetaSingletonABC(SingletonMetaClass, ABCMeta):\n \"\"\"\n Metaclass that inherits both SingletonMetaClass, and ABCMeta \\\n (collection.Mappings' metaclass).\n \"\"\"\n pass\n\n\nclass ConfigRegistry(Mapping, metaclass=MetaSingletonABC):\n \"\"\"\n Registry singleton for the parameters loaded from the configuration files.\n\n Registers `default` values (coming from workers, models, tasks, etc) as well as \\\n `config` values loaded by the user for a particular experiment.\n\n Parameters can be read from the registry by indexing.\n The returned parameters are the `default` ones superseded by all the `config` ones.\n\n The merging of `default` and `config` parameters is computed every time the registry is changed.\n\n Can contain nested parameters sections (acts as a dict).\n\n .. warning::\n\n This class should not be used except through :py:class:`ConfigInterface`.\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Constructor:\n\n - Call base constructor (:py:class:`Mapping`),\n - Initializes empty parameters dicts for:\n\n - `Default` parameters,\n - `Config` parameters,\n - Resulting tree.\n\n\n \"\"\"\n super(ConfigRegistry, self).__init__()\n # Default parameters set in the code.\n\n self._clear_registry()\n\n def _clear_registry(self):\n \"\"\"\n Removes the content of the registry\n\n .. warning::\n Use with caution!\n \"\"\"\n self._default_params = {}\n # Parameters read from configuration files.\n self._superseding_config_params = {}\n # Resulting parameters.\n self._params = {}\n\n\n def _update_params(self):\n \"\"\"\n Update the resulting parameters dict from the `default` parameters dict superseded by the \\\n `config` params registry.\n\n \"\"\"\n # deep copy to avoid the config params leaking to `self._default_params`\n self._params = copy.deepcopy(self._default_params)\n self.update_dict_recursively(self._params, self._superseding_config_params)\n\n def add_default_params(self, default_params: dict):\n \"\"\"\n Appends ``default_params`` to the `default` parameter dict of the current :py:class:`ConfigRegistry`, \\\n and update the resulting parameters dict.\n\n .. note::\n\n This method should be used by the objects necessitating default values \\\n (tasks, models, workers etc.).\n\n :param default_params: Dictionary containing default values.\n :type default_params: dict\n\n \"\"\"\n # Update default params list.\n self.update_dict_recursively(self._default_params, default_params)\n # Merge default with config list.\n self._update_params()\n\n def add_config_params(self, config_params: dict):\n \"\"\"\n Appends ``config_params`` to the `config` parameter dict of the current :py:class:`ConfigRegistry`, \\\n and update the resulting parameters dict.\n\n .. note::\n\n This is intended for the user to dynamically (re)configure his experiments.\n\n :param config_params: Dictionary containing config values.\n :type config_params: dict\n\n \"\"\"\n # Update config params list.\n self.update_dict_recursively(self._superseding_config_params, config_params)\n # Merge default with config list.\n self._update_params()\n\n def del_default_params(self, keypath: list):\n \"\"\"\n Removes an entry from the `default` parameter dict of the current :py:class:`ConfigRegistry`, \\\n and update the resulting parameters dict.\n\n The entry can either be a subtree or a leaf of the `default` parameter dict.\n\n :param keypath: list of keys to subtree / leaf in the `default` parameter dict.\n :type keypath: list\n\n \"\"\"\n self.delete_subtree(self._default_params, keypath)\n self._update_params()\n\n def del_config_params(self, keypath: list):\n \"\"\"\n Removes an entry from the `config` parameter dict of the current :py:class:`ConfigRegistry`, \\\n and update the resulting parameters dict.\n\n The entry can either be a subtree or a leaf of the `config` parameter dict.\n\n :param keypath: list of keys to subtree / leaf in the `config` parameter dict.\n :type keypath: list\n\n \"\"\"\n self.delete_subtree(self._superseding_config_params, keypath)\n self._update_params()\n\n def __getitem__(self, key):\n \"\"\"\n Get parameter value under ``key``.\n\n The parameter dict is derived from the default parameters updated with the config parameters.\n\n :param key: key to value in the :py:class:`ConfigRegistry`.\n :type key: str\n\n :return: Parameter value\n\n \"\"\"\n return self._params[key]\n\n def __iter__(self):\n \"\"\"\n\n :return: Iterator over the :py:class:`ConfigRegistry`.\n\n \"\"\"\n return iter(self._params)\n\n def __len__(self):\n \"\"\"\n\n :return: Length of the :py:class:`ConfigRegistry`.\n\n \"\"\"\n return len(self._params)\n\n def __eq__(self, other):\n \"\"\"\n Check whether two registrys are equal (just for the purpose of compatibility with base Mapping class).\n \"\"\"\n if isinstance(other, self.__class__):\n # As this is singleton class - the following is always True.\n return self._params == other._params\n else:\n return False\n\n \n def update_dict_recursively(self, current_node, update_node):\n \"\"\"\n Recursively update the ``current_node`` of the :py:class:`ConfigRegistry` with the values of \\\n the ``update_node``.\n\n Starts from the root of the ``current_node``.\n\n :param current_node: Current (default or config) node.\n :type current_node: :py:class:`ConfigRegistry` (inheriting from :py:class:`Mapping`)\n\n :param update_node: Values to be added/updated to the ``current_node``.\n :type update_node: :py:class:`ConfigRegistry` (inheriting from :py:class:`Mapping`)\n\n :return: Updated current node.\n\n \"\"\"\n for k, v in update_node.items():\n if isinstance(v, Mapping):\n # Make sure that key exists and it is not None.\n if k not in current_node.keys() or current_node[k] is None:\n current_node[k] = {}\n current_node[k] = self.update_dict_recursively(current_node[k], v)\n else:\n current_node[k] = v\n return current_node\n\n @staticmethod\n def delete_subtree(current_dict, keypath: list):\n \"\"\"\n Delete the subtree indexed by the ``keypath`` from the ``current_dict``.\n\n :param current_dict: dictionary to act on.\n :type current_dict: dict\n\n :param keypath: list of keys to subtree in ``current_dict`` to delete\n :type keypath: list\n \"\"\"\n if len(keypath) < 1:\n raise KeyError\n\n def lookup_recursion(dic, key, *keys):\n if keys:\n return lookup_recursion(dic[key], *keys)\n return dic[key]\n\n lookup_keys = keypath[:-1] # We keep the last key for use with `del`\n if len(lookup_keys) > 0:\n r = lookup_recursion(current_dict, *lookup_keys)\n del r[keypath[-1]]\n else:\n del current_dict[keypath[-1]]\n" }, { "path": "ptp/configuration/configuration_error.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM tkornuta, Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nclass ConfigurationError(Exception):\n \"\"\" Error thrown when encountered a configuration issue. \"\"\"\n def __init__(self, msg):\n \"\"\" Stores message \"\"\"\n self.msg = msg\n\n def __str__(self):\n \"\"\" Prints the message \"\"\"\n return repr(self.msg)\n\n" }, { "path": "ptp/data_types/__init__.py", "content": "from .data_streams import DataStreams\nfrom .data_definition import DataDefinition\n\n__all__ = [\n 'DataStreams',\n 'DataDefinition',\n ]\n" }, { "path": "ptp/data_types/data_definition.py", "content": "\n#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom collections import namedtuple\n\n\nclass DataDefinition(namedtuple(\"DataDefinition\", 'dimensions types description')):\n __slots__ = ()\n" }, { "path": "ptp/data_types/data_streams.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta, Vincent Marois\"\n\nimport torch\nimport collections\n\nclass DataStreams(collections.abc.MutableMapping):\n \"\"\"\n - Mapping: A container object that supports arbitrary key lookups and implements the methods ``__getitem__``, \\\n ``__iter__`` and ``__len__``.\n\n - Mutable objects can change their value but keep their id() -> ease modifying existing keys' value.\n\n DataStreams: Dict used for storing batches of data by tasks.\n\n **This is the main object class used to share data between all components through a worker, starting from task to loss and visualization.**\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n \"\"\"\n DataStreams constructor. Can be initialized in different ways:\n\n >>> data_streams = DataStreams()\n >>> data_streams = DataStreams({'inputs': torch.tensor(), 'targets': numpy.ndarray()})\n >>> # etc.\n\n :param args: Used to pass a non-keyworded, variable-length argument list.\n\n :param kwargs: Used to pass a keyworded, variable-length argument list.\n \"\"\"\n self.__dict__.update(*args, **kwargs)\n\n\n def __setitem__(self, key, value, addkey=False):\n \"\"\"\n key:value setter function.\n\n :param key: Dict Key.\n\n :param value: Associated value.\n\n :param addkey: Indicate whether or not it is authorized to add a new key `on-the-fly`.\\\n Default: ``False``.\n :type addkey: bool\n\n .. warning::\n\n `addkey` is set to ``False`` by default as setting it to ``True`` removes the constraints of the\\\n ``DataStreams`` and enables it to become mutable.\n \"\"\"\n if not addkey and key not in self.keys():\n msg = 'Cannot modify a non-existing key \"{}\" in DataStreams'.format(key)\n raise KeyError(msg)\n else:\n self.__dict__[key] = value\n\n\n def publish(self, dict_to_add):\n \"\"\"\n Publishes a new data streams - extends data stream object by adding (keys,values) from data_definitions.\n\n .. warning::\n This is in-place operation, i.e. extends existing object, does not return a new one.\n\n :param data_streams: :py:class:`ptp.utils.DataStreams` object to be extended.\n\n :param data_definitions: key-value pairs.\n\n \"\"\"\n for (key,value) in dict_to_add.items():\n if key in self.keys():\n msg = \"Cannot extend DataStreams, as {} already present in its keys\".format(key)\n raise KeyError(msg)\n # Call setitem with \"additional argument\".\n self.__setitem__(key, value, addkey=True)\n\n\n def reinitialize(self, streams_to_leave):\n \"\"\"\n Removes all streams (keys and associated values) from DataStreams EXCEPT the ones passed in ``streams_to_leave``.\n \"\"\"\n # Keys to remove.\n rem_keys = [key for key in self.keys() if key not in streams_to_leave.keys()]\n # Leave index.\n if 'index' in rem_keys:\n rem_keys.remove('index')\n # Remove.\n for key in rem_keys:\n self.__delitem__(key, delkey=True)\n\n\n def __getitem__(self, key):\n \"\"\"\n Value getter function.\n\n :param key: Dict Key.\n\n :return: Associated Value.\n\n \"\"\"\n return self.__dict__[key]\n\n def __delitem__(self, key, delkey=False):\n \"\"\"\n Delete a key:value pair.\n\n :param delkey: Indicate whether or not it is authorized to add a delete the key `on-the-fly`.\\\n Default: ``False``.\n :type delkey: bool\n\n .. warning::\n\n By default, it is not authorized to delete an existing key. Set `delkey` to ``True`` to ignore this\\\n restriction and \n\n :param key: Dict Key.\n\n \"\"\"\n if not delkey:\n msg = 'Cannot delete key \"{}\" from DataStreams'.format(key)\n raise KeyError(msg)\n else:\n del self.__dict__[key]\n\n def __iter__(self):\n return iter(self.__dict__)\n\n def __len__(self):\n return len(self.__dict__)\n\n def __str__(self):\n \"\"\"\n :return: A simple Dict representation of ``DataStreams``.\n\n \"\"\"\n return str(self.__dict__)\n\n def __repr__(self):\n \"\"\"\n :return: Echoes class, id, & reproducible representation in the Read–Eval–Print Loop.\n\n \"\"\"\n return '{}, DataStreams({})'.format(super(DataStreams, self).__repr__(), self.__dict__)\n\n\n def to(self, device=None, keys_to_move=None, non_blocking=False):\n \"\"\"\n Moves object(s) to device\n\n .. note::\n\n Wraps call to ``torch.Tensor.to()``: If this object is already in CUDA memory and on the correct device, \\\n then no copy is performed and the original object is returned.\n If an element of `self` is not a ``torch.tensor``, it is returned as is, \\\n i.e. We only move the ``torch.tensor`` (s) contained in `self`. \\\n\n\n :param device: The destination GPU device. Defaults to the current CUDA device.\n :type device: torch.device\n\n :param non_blocking: If True and the source is in pinned memory, the copy will be asynchronous with respect to \\\n the host. Otherwise, the argument has no effect. Default: ``False``.\n :type non_blocking: bool\n\n \"\"\"\n for key in self:\n if isinstance(self[key], torch.Tensor):# and (not self[key].is_cuda):\n # Skip keys that are not in the keys_to_move list (if it was passed).\n if keys_to_move is not None and key not in keys_to_move:\n continue\n self[key] = self[key].to(device=device)#, non_blocking=non_blocking)" }, { "path": "ptp/utils/__init__.py", "content": "from .app_state import AppState\nfrom .data_streams_parallel import DataStreamsParallel\nfrom .globals_facade import GlobalsFacade\nfrom .key_mappings_facade import KeyMappingsFacade\nfrom .samplers import kFoldRandomSampler\nfrom .samplers import kFoldWeightedRandomSampler\nfrom .singleton import SingletonMetaClass\nfrom .statistics_aggregator import StatisticsAggregator\nfrom .statistics_collector import StatisticsCollector\nfrom .termination_condition import TerminationCondition\n\n\n__all__ = [\n 'AppState',\n 'DataStreamsParallel',\n 'GlobalsFacade',\n 'KeyMappingsFacade',\n 'kFoldRandomSampler',\n 'kFoldWeightedRandomSampler',\n 'SingletonMetaClass',\n 'StatisticsAggregator',\n 'StatisticsCollector', \n 'TerminationCondition',\n ]\n" }, { "path": "ptp/utils/app_state.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n__author__ = \"Alexis Asseman, Tomasz Kornuta\"\n\nimport torch\n\nfrom os import path\n\nfrom ptp.utils.singleton import SingletonMetaClass\n\n\nclass AppState(metaclass=SingletonMetaClass):\n \"\"\"\n Represents the application state. A singleton that can be accessed by calling:\n\n >>> app_state = AppState()\n\n Contains global variables that can be accessed with standard setted/getter methods:\n\n >>> app_state[\"test1\"] = 1 \n >>> app_state[\"test2\"] = 2\n >>> print(app_state[\"test1\"])\n\n .. warning::\n It is assumed that global variables are immutable, i.e. once a variable is set, it cannot be changed \n\n >>> app_state[\"test1\"] = 3 # Raises AtributeError\n\n Additionally, it stores all properly parsed commandline arguments.\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Constructor. Initializes dictionary with global variables, sets CPU types as default.\n\n \"\"\"\n # Empty commandline arguments.\n self.args = None\n\n # Field storing global variables.\n self.__globals = dict()\n\n # Get absolute path to configs from \"~/./ptp/configs\".\n ptp_path = path.expanduser(\"~/.ptp/\")\n with open(path.join(ptp_path, \"config.txt\")) as file:\n self.absolute_config_path = file.readline()\n\n # Initialize logger logfile (as empty for now).\n self.log_file = None\n self.logger = None\n # Set default path to current dir.\n self.log_dir = path.expanduser(\".\")\n\n # Set CPU types as default.\n self.set_cpu_types()\n self.use_gpu = False\n self.use_dataparallel = False\n self.device = torch.device('cpu')\n\n # Reset global counters.\n self.epoch = None # Processor is not using the notion of epoch.\n self.episode = 0\n\n\n def set_types(self):\n \"\"\"\n Enables computations on CUDA if GPU is available.\n Sets the default data types.\n \"\"\"\n # Determine if GPU/CUDA is available.\n if torch.cuda.is_available() and self.args.use_gpu:\n self.logger.info('Running computations on GPU using CUDA')\n self.set_gpu_types()\n # Use GPU.\n self.use_gpu = True\n self.device = torch.device('cuda')\n # Use DataParallel if more than 1 device is available.\n if torch.cuda.device_count() > 1:\n self.use_dataparallel = True\n elif self.args.use_gpu:\n self.logger.warning('GPU utilization is demanded but there are no available GPU devices! Using CPUs instead')\n else:\n self.logger.info('GPU utilization is disabled, performing all computations on CPUs')\n\n\n def set_cpu_types(self):\n \"\"\"\n Sets all tensor types to CPU data types.\n \"\"\"\n self.FloatTensor = torch.FloatTensor\n self.DoubleTensor = torch.DoubleTensor\n self.HalfTensor = torch.HalfTensor\n self.ByteTensor = torch.ByteTensor\n self.CharTensor = torch.CharTensor\n self.ShortTensor = torch.ShortTensor\n self.IntTensor = torch.IntTensor\n self.LongTensor = torch.LongTensor\n\n\n def set_gpu_types(self):\n \"\"\"\n Sets all tensor types to GPU/CUDA data types.\n \"\"\"\n self.FloatTensor = torch.cuda.FloatTensor\n self.DoubleTensor = torch.cuda.DoubleTensor\n self.HalfTensor = torch.cuda.HalfTensor\n self.ByteTensor = torch.cuda.ByteTensor\n self.CharTensor = torch.cuda.CharTensor\n self.ShortTensor = torch.cuda.ShortTensor\n self.IntTensor = torch.cuda.IntTensor\n self.LongTensor = torch.cuda.LongTensor\n\n\n def globalkeys(self):\n \"\"\"\n Yields global keys.\n \"\"\"\n for key in self.__globals.keys():\n yield key\n\n\n def globalitems(self):\n \"\"\"\n Yields global keys.\n \"\"\"\n for key,value in self.__globals.items():\n yield key,value\n\n\n def __setitem__(self, key, value, override=False):\n \"\"\"\n Adds global variable. \n\n :param key: Dict Key.\n\n :param value: Associated value.\n\n :param override: Indicate whether or not it is authorized to override the existing key.\\\n Default: ``False``.\n :type override: bool\n\n .. warning::\n Once global variable is set, its value cannot be changed (it becomes immutable).\n \"\"\"\n if not override and key in self.__globals.keys():\n if (self.__globals[key] != value):\n raise KeyError(\"Global key '{}' already exists and has different value (existing {} vs received {})!\".format(key, self.__globals[key], value))\n #msg = 'Cannot add or modify key \"{}\" as it is already present in global variables'.format(key)\n #raise KeyError(msg)\n else:\n self.__globals[key] = value\n\n\n def __getitem__(self, key):\n \"\"\"\n Value getter function.\n\n :param key: Dict Key.\n\n :return: Associated Value.\n \"\"\"\n if key not in self.__globals.keys():\n msg = \"Key '{}' not present in global variables\".format(key)\n raise KeyError(msg)\n else:\n return self.__globals[key]\n" }, { "path": "ptp/utils/data_streams_parallel.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nimport torch\n\nfrom torch.nn.parallel._functions import Scatter, Gather\nfrom torch.nn.parallel.replicate import replicate\nfrom torch.nn.parallel.parallel_apply import parallel_apply\n\n\nfrom ptp.data_types.data_streams import DataStreams\n\n\ndef data_streams_scatter(inputs, target_gpus, dim=0):\n r\"\"\"\n Slices tensors into approximately equal chunks and\n distributes them across given GPUs. Duplicates\n references to objects that are not tensors.\n \"\"\"\n def scatter_map(obj):\n if isinstance(obj, torch.Tensor):\n return Scatter.apply(target_gpus, None, dim, obj)\n if isinstance(obj, tuple) and len(obj) > 0:\n return list(zip(*map(scatter_map, obj)))\n if isinstance(obj, list) and len(obj) > 0:\n return list(map(list, zip(*map(scatter_map, obj))))\n if isinstance(obj, dict) and len(obj) > 0:\n return list(map(type(obj), zip(*map(scatter_map, obj.items()))))\n if isinstance(obj, DataStreams) and len(obj) > 0:\n return list(map(type(obj), zip(*map(scatter_map, obj.items())))) \n # Return \"unscattered\" object for all GPUs.\n # This seems to be the cause of the issue for SentenceEmbeddings!\n # TODO: further investigate.\n return [obj for _ in target_gpus]\n\n # After scatter_map is called, a scatter_map cell will exist. This cell\n # has a reference to the actual function scatter_map, which has references\n # to a closure that has a reference to the scatter_map cell (because the\n # fn is recursive). To avoid this reference cycle, we set the function to\n # None, clearing the cell\n try:\n return scatter_map(inputs)\n finally:\n scatter_map = None\n\n\ndef data_streams_scatter_kwargs(inputs, kwargs, target_gpus, dim=0):\n r\"\"\"Scatter with support for kwargs dictionary\"\"\"\n inputs = data_streams_scatter(inputs, target_gpus, dim) if inputs else []\n kwargs = data_streams_scatter(kwargs, target_gpus, dim) if kwargs else []\n if len(inputs) < len(kwargs):\n inputs.extend([() for _ in range(len(kwargs) - len(inputs))])\n elif len(kwargs) < len(inputs):\n kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))])\n inputs = tuple(inputs)\n kwargs = tuple(kwargs)\n return inputs, kwargs\n\n\ndef data_streams_gather(outputs, target_device, dim=0):\n r\"\"\"\n Gathers tensors from different GPUs on a specified device\n (-1 means the CPU).\n \"\"\"\n def gather_map(outputs):\n out = outputs[0]\n if isinstance(out, torch.Tensor):\n return Gather.apply(target_device, dim, *outputs)\n if out is None:\n return None\n\n if isinstance(out, DataStreams):\n if not all((len(out) == len(d) for d in outputs)):\n raise ValueError('All dicts must have the same number of keys')\n return type(out)(((k, gather_map([d[k] for d in outputs]))\n for k in out))\n\n if isinstance(out, dict):\n if not all((len(out) == len(d) for d in outputs)):\n raise ValueError('All dicts must have the same number of keys')\n return type(out)(((k, gather_map([d[k] for d in outputs]))\n for k in out))\n\n return type(out)(map(gather_map, zip(*outputs)))\n\n # Recursive function calls like this create reference cycles.\n # Setting the function to None clears the refcycle.\n try:\n return gather_map(outputs)\n finally:\n gather_map = None\n\n\nclass DataStreamsParallel(torch.nn.DataParallel):\n \"\"\"\n Modified DataParallel wrapper enabling operation on DataStreamss.\n \n .. warning:\n Compatible with PyTorch v1.0.1 !!\n\n \"\"\"\n def __init__(self, module, device_ids=None, output_device=None, dim=0):\n super(DataStreamsParallel, self).__init__(module, device_ids, output_device, dim)\n\n def forward(self, *inputs, **kwargs):\n \"\"\"\n Performs \"parallelized forward\" pass by scattering batch into several batches, distributing models on different GPUs, performing parallel pass and gathering results into a single (returned) DataStreams.\n\n ..warning:\n As the \"external\" operations are changing inputs to tuple of DataStreamss, extension of main DataStreams must be done \"outside\" of this method.\n \"\"\"\n\n # Simple processing.\n if not self.device_ids:\n return self.module(*inputs, **kwargs)\n # One device - also easy.\n if len(self.device_ids) == 1:\n return self.module(*inputs[0], **kwargs[0])\n\n # Preprocessing: get only the inputs important for to the wrapped model (optimization).\n inputs_tuple = []\n for i, item in enumerate(inputs):\n input_dict = DataStreams({key: value for key,value in item.items() if key in self.module.input_data_definitions().keys()})\n inputs_tuple.append(input_dict)\n # Convert to tuple.\n inputs_tuple = tuple(inputs_tuple)\n\n # Scatter inputs into several tuples.\n inputs_tuple, kwargs = self.scatter(inputs_tuple, kwargs, self.device_ids)\n\n # Create replicas of the module on all devices.\n replicas = self.replicate(self.module, self.device_ids[:len(inputs_tuple)])\n\n # Pass scattered inputs throught those replicas.\n self.parallel_apply(replicas, inputs_tuple, kwargs)\n\n # Gather tuple. This cannot be done \"in place\"!\n gathered_tuple = self.gather(inputs_tuple, self.output_device)\n\n # Return 0-th tuple, i.e. a single DataStreams on device 0.\n return gathered_tuple[0]\n\n\n def replicate(self, module, device_ids):\n return replicate(module, device_ids)\n\n def scatter(self, inputs, kwargs, device_ids):\n return data_streams_scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)\n\n def parallel_apply(self, replicas, inputs, kwargs):\n return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])\n\n def gather(self, outputs, output_device):\n return data_streams_gather(outputs, output_device, dim=self.dim)\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds statistics for the wrapped model.\n\n :param stat_col: ``StatisticsCollector``.\n \"\"\"\n self.module.add_statistics(stat_col)\n\n\n def collect_statistics(self, stat_col, data_streams):\n \"\"\"\n Collects statistics for the wrapped model.\n\n :param stat_col: :py:class:`ptp.utils.StatisticsCollector`.\n\n :param data_streams: ``DataStreams`` containing inputs, targets etc.\n :type data_streams: :py:class:`ptp.data_types.DataStreams`\n \"\"\"\n self.module.collect_statistics(stat_col, data_streams)\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Aggregates statistics for the wrapped model.\n\n :param stat_agg: ``StatisticsAggregator``.\n \"\"\"\n self.module.add_aggregators(stat_agg)\n\n\n def aggregate_statistics(self, stat_col, stat_agg):\n \"\"\"\n Aggregates statistics for the wrapped model.\n\n :param stat_col: ``StatisticsCollector``\n\n :param stat_agg: ``StatisticsAggregator``\n \"\"\"\n self.module.aggregate_statistics(stat_col, stat_agg)\n" }, { "path": "ptp/utils/globals_facade.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom ptp.utils.app_state import AppState\n\nclass GlobalsFacade(object):\n \"\"\"\n Simple facility for accessing global variables using provided mappings using list-like read-write access.\n \"\"\"\n def __init__(self, key_mappings):\n \"\"\"\n Constructor. Initializes app state and stores key mappings.\n\n :param key_mappings: Dictionary of global key mappings of the parent object.\n \"\"\"\n # Remember parent object global keys mappings.\n self.key_mappings = key_mappings\n self.app_state = AppState()\n\n def __setitem__(self, key, value):\n \"\"\"\n Sets global value using parent object key mapping.\n\n :param key: Global key name (that will be mapped).\n\n :param value: Value that will be set.\n \"\"\"\n # Retrieve key using parent object global key mappings.\n mapped_key = self.key_mappings.get(key, key)\n # Set global balue.\n self.app_state[mapped_key] = value\n\n\n def __getitem__(self, key):\n \"\"\"\n Global value getter function.\n Uses parent object key mapping for accesing the value.\n\n :param key: Global key name (that will be mapped).\n\n :return: Associated Value.\n \"\"\"\n # Retrieve key using parent object global key mappings.\n mapped_key = self.key_mappings.get(key, key)\n # Retrieve the value.\n return self.app_state[mapped_key]" }, { "path": "ptp/utils/key_mappings_facade.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nclass KeyMappingsFacade(object):\n \"\"\"\n Simple facility for accessing key names using provided mappings using list-like (read-only) access.\n \"\"\"\n def __init__(self, key_mappings):\n \"\"\"\n Constructor. Stores key mappings.\n\n :param key_mappings: Dictionary of key mappings of the parent object.\n \"\"\"\n # Remember parent object global keys.\n self.keys_mappings = key_mappings\n\n def __getitem__(self, key):\n \"\"\"\n Global value getter function.\n Uses parent object key mapping for accesing the value.\n\n :param key: Global key name (that will be mapped).\n\n :return: Associated Value.\n \"\"\"\n # Retrieve key using parent object global key mappings.\n return self.keys_mappings.get(key, key)\n" }, { "path": "ptp/utils/logger.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport logging\nimport logging.config as logging_config\n\nfrom ptp.utils.app_state import AppState\n\ndef initialize_logger(name, add_file_handler = True):\n \"\"\"\n Initializes the logger, with a specific configuration.\n Requires that AppState has the following variable already set:\n - AppState().args.log_level -- log level (from command line argsuments)\n\n :param name: Name of the entity that \"owns\" the logger.\n\n :return: Logger object.\n\n \"\"\"\n # Load the default logger configuration.\n logger_config = {'version': 1,\n 'disable_existing_loggers': False,\n 'formatters': {\n 'simple': {\n 'format': '[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',\n 'datefmt': '%Y-%m-%d %H:%M:%S'}},\n 'handlers': {\n 'console': {\n 'class': 'logging.StreamHandler',\n 'level': 'INFO',\n 'formatter': 'simple',\n 'stream': 'ext://sys.stdout'}},\n 'root': {'level': 'DEBUG',\n 'handlers': ['console']}}\n\n logging_config.dictConfig(logger_config)\n\n # Create the Logger, set its label and logging level.\n logger = logging.getLogger(name=name)\n\n # Add file handler - when the file is initialized...\n if add_file_handler:\n add_file_handler_to_logger(logger)\n\n # Set logger level depending on the settings.\n if AppState().args is not None and AppState().args.log_level is not None:\n logger.setLevel(getattr(logging, AppState().args.log_level.upper(), None))\n else:\n logger.setLevel('INFO')\n\n return logger\n\n\ndef add_file_handler_to_logger(logger):\n \"\"\"\n Add a ``logging.FileHandler`` to the logger.\n Requires that AppState has the following variable already set:\n\n - AppState().logfile - name (with path) of the file that everything will be used as destination by the ``FileHandler``.\n\n :param logger: Logger object.\n\n \"\"\"\n # This makes \n if AppState().log_file is None:\n return\n\n # Create file handler which logs even DEBUG messages.\n fh = logging.FileHandler(AppState().log_file)\n\n # Set logging level for this file.\n fh.setLevel(logging.DEBUG)\n\n # Create formatter and add it to the handlers.\n formatter = logging.Formatter(fmt='[%(asctime)s] - %(levelname)s - %(name)s >>> %(message)s',\n datefmt='%Y-%m-%d %H:%M:%S')\n fh.setFormatter(formatter)\n\n # Add the handler to the logger.\n logger.addHandler(fh)\n" }, { "path": "ptp/utils/samplers.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nfrom math import ceil\n\nimport torch\nfrom torch._six import int_classes as _int_classes\nfrom torch.utils.data.sampler import Sampler\n\nclass kFoldRandomSampler(Sampler):\n \"\"\"\n Samples indices using the k-fold cross validation approach.\n Offers two modes:\n - generate indices for all-but-one folds (for training)\n - generate indices for only one fold (for validation)\n \n Every time __iter__() method is called, it moves to next fold/set of folds. \n \"\"\"\n\n def __init__(self, num_samples, num_folds, epochs_per_fold = 1, all_but_current_fold = True):\n \"\"\"\n Initializes the sampler by generating the indices associated with the fold(s) that are to be used.\n\n :param num_samples: Size of the dataset\n\n :param num_folds: Number of folds\n\n :param epochs_per_fold: Number of epochs that need to pass before sampler moves to next fold(s) (DEFAULT: 1)\n\n :param all_but_current_fold: Operation mode (DEFAULT: True): \\\n When True, generates indices for all-but-one folds (for training) \\\n When False, generates indices for only one fold (for validation)\n \"\"\"\n # Get number of samples (size of \"whole dataset\").\n if not isinstance(num_samples, _int_classes) or isinstance(num_samples, bool) or \\\n num_samples <= 0:\n raise ValueError(\"num_samples should be a positive integeral \"\n \"value, but got num_samples={}\".format(num_samples))\n self.num_samples = num_samples\n\n # Get number of folds.\n if not isinstance(num_folds, _int_classes) or isinstance(num_samples, bool) or \\\n num_folds <= 0:\n raise ValueError(\"num_folds should be a positive integeral \"\n \"value, but got num_folds={}\".format(num_folds))\n\n # Get number epochs per fold.\n if not isinstance(epochs_per_fold, _int_classes) or isinstance(epochs_per_fold, bool) or \\\n epochs_per_fold <= 0:\n raise ValueError(\"epochs_per_fold should be a positive integeral \"\n \"value, but got num_folds={}\".format(epochs_per_fold))\n\n # Store fold-related parameres.\n self.all_but_current_fold = all_but_current_fold\n self.num_folds = num_folds\n self.epochs_per_fold = epochs_per_fold\n\n # Initialize current \"fold\" so it will return samples for 0-th fold/all-but-0th fold.\n self.current_fold = 0\n # \"Left epochs\": +1 is related to \"initial\", additional generation of indices - below.\n self.epochs_left = self.epochs_per_fold +1\n\n # Generate \"initial\" indices.\n self.indices = self.regenerate_indices()\n\n def regenerate_indices(self):\n \"\"\"\n (Re)generates indices used by sampler repending of mode and given fold number.\n\n :return: (re)generated indices.\n \"\"\"\n # Fold size and indices.\n all_indices = range(self.num_samples)\n fold_size = ceil(self.num_samples / self.num_folds)\n fold = self.current_fold\n\n # Generate indices associated with the given fold / all except the given fold.\n if self.all_but_current_fold:\n if fold == 0:\n first = (fold+1)*fold_size\n # Create indices set.\n return all_indices[first:]\n else:\n # Concatenate two subsets of indices.\n first_0 = 0\n # All samples aside of those between last_0 and first_1.\n last_0 = fold*fold_size\n first_1 = (fold+1)*fold_size\n # Take the rest.\n last_1 = self.num_samples\n # Create indices set from two subsets.\n return [*all_indices[first_0:last_0], *all_indices[first_1:last_1]]\n else:\n # Get first/last indices.\n first = fold*fold_size\n # Assume that the last fold might be \"smaller\".\n last = min((fold+1)*fold_size, self.num_samples)\n # Create indices set.\n return all_indices[first:last]\n\n\n def __iter__(self):\n \"\"\"\n Return \"shuffled\" indices.\n \"\"\"\n # \"Decrease\" the number of epochs with this fold.\n self.epochs_left = self.epochs_left - 1\n if self.epochs_left <= 0:\n # Next fold, modulo by the total number of folds.\n self.current_fold = (self.current_fold + 1) % self.num_folds\n\n # Regenerate indices.\n self.indices = self.regenerate_indices()\n\n # Reset epochs counter.\n self.epochs_left = self.epochs_per_fold\n\n # Return permutated indices.\n return (self.indices[i] for i in torch.randperm(len(self.indices)))\n\n def __len__(self):\n \"\"\"\n Return length of dataset.\n \"\"\"\n return len(self.indices)\n\n\nclass kFoldWeightedRandomSampler(kFoldRandomSampler):\n \"\"\"\n Samples indices using the k-fold cross validation, additionally relying on the provided probabilities (weights).\n Offers two modes:\n - generate indices for all-but-one folds (for training)\n - generate indices for only one fold (for validation)\n \n Every time __iter__() method is called, it moves to next fold/set of folds. \n \"\"\"\n\n def __init__(self, weights, num_samples, num_folds, epochs_per_fold = 1, all_but_current_fold = True, replacement=True):\n \"\"\"\n Initializes the sampler by generating the indices associated with the fold(s) that are to be used.\n\n :param num_samples: Size of the dataset \n\n :param num_folds: Number of folds\n\n :param epochs_per_fold: Number of epochs that need to pass before sampler moves to next fold(s) (DEFAULT: 1)\n\n :param all_but_current_fold: Operation mode (DEFAULT: True): \\\n When True, generates indices for all-but-one folds (for training) \\\n When False, generates indices for only one fold (for validation)\n\n :params weights: a sequence of weights, not necessary summing up to one\n\n :param num_samples: number of samples to draw\n\n :param replacement: if ``True``, samples are drawn with replacement.\n If not, they are drawn without replacement, which means that when a\n sample index is drawn for a row, it cannot be drawn again for that row.\n \"\"\"\n # Call k-fold base class constructor.\n super().__init__(num_samples, num_folds, epochs_per_fold, all_but_current_fold)\n # Get replacement flag.\n if not isinstance(replacement, bool):\n raise ValueError(\"replacement should be a boolean value, but got \"\n \"replacement={}\".format(replacement))\n self.replacement = replacement\n\n # Get weights.\n self.weights = torch.tensor(weights, dtype=torch.double)\n\n def __iter__(self):\n # \"Decrease\" the number of epochs with this fold.\n self.epochs_left = self.epochs_left - 1\n if self.epochs_left <= 0:\n # Next fold, modulo by the total number of folds.\n self.current_fold = (self.current_fold + 1) % self.num_folds\n\n # Regenerate indices.\n self.indices = self.regenerate_indices()\n\n # Reset epochs counter.\n self.epochs_left = self.epochs_per_fold\n\n # Select the corresponging weights.\n weights = torch.take(self.weights, torch.tensor(self.indices))\n \n # Return indices sampled with multinomial distribution.\n return (self.indices[i] for i in torch.multinomial(weights, len(self.indices), self.replacement).tolist())\n" }, { "path": "ptp/utils/singleton.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Alexis Asseman\"\n\nclass SingletonMetaClass(type):\n _instances = {}\n\n def __call__(cls, *args, **kwargs):\n if cls not in cls._instances:\n cls._instances[cls] = super(\n SingletonMetaClass, cls).__call__(*args, **kwargs)\n return cls._instances[cls]\n" }, { "path": "ptp/utils/statistics_aggregator.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Vincent Marois, Tomasz Kornuta\"\n\nfrom ptp.utils.statistics_collector import StatisticsCollector\n\n\nclass StatisticsAggregator(StatisticsCollector):\n \"\"\"\n Specialized class used for the computation of several statistical aggregators.\n\n Inherits from :py:class:`ptp.utils.StatisticsCollector` as it extends its capabilities: it relies \\\n on :py:class:`ptp.utils.StatisticsCollector` to collect the statistics over an epoch (training set) \\\n or a validation (over the validation set).\n\n Once the statistics have been collected, the :py:class:`ptp.utils.StatisticsAggregator` allows \\\n to compute several statistical aggregators to summarize the last epoch or validation phase.\n\n E.g. With the list of loss values from the last epoch, we can compute the average loss, the min & max, \\\n and the standard deviation.\n\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Constructor for the :py:class:`ptp.utils.StatisticsAggregator`. Defines empty aggregators dict.\n\n Other statistical aggregators can be added via :py:func:`StatisticsAggregator.add_aggregator()`.\n\n \"\"\"\n # call base constructor\n super(StatisticsAggregator, self).__init__()\n\n self.aggregators = dict()\n\n def add_aggregator(self, key, formatting):\n \"\"\"\n Add a statistical aggregator.\n The value associated to the specified key is initiated as -1.\n\n :param key: Statistical aggregator to add. Such aggregator (e.g. min, max, mean, std...)\\\n should be based on an existing statistics collected by the :py:class:`ptp.utils.StatisticsCollector` \\\n (e.g. added by :py:func:`StatisticsCollector.add_statistics()` and collected by \\\n :py:func:`ptp.models.Model.collect_statistics()` or \\\n :py:func:`ptp.models.Task.collect_statistics()`.\n :type key: str\n\n :param formatting: Formatting that will be used when logging and exporting to CSV.\n :type formatting: str\n\n \"\"\"\n self.formatting[key] = formatting\n\n # instantiate associated value as list.\n self.aggregators[key] = -1\n\n def __getitem__(self, key):\n \"\"\"\n Get the values list of the specified statistical aggregator.\n\n :param key: Name of the statistical aggregator to get the values list of.\n :type key: str\n\n :return: Values list associated with the specified statistical aggregator.\n\n \"\"\"\n return self.aggregators[key]\n\n def __setitem__(self, key, value):\n \"\"\"\n Set the value of the specified statistical aggregator, thus overwriting the existing one.\n\n :param key: Name of the statistical aggregator to set the value of.\n :type key: str\n\n :param value: Value to set for the given key.\n :type value: int, float\n\n \"\"\"\n self.aggregators[key] = value\n\n def __delitem__(self, key):\n \"\"\"\n Delete the specified statistical aggregator.\n\n :param key: Key to be deleted.\n :type key: str\n\n \"\"\"\n del self.aggregators[key]\n\n def __len__(self):\n \"\"\"\n Returns the number of tracked statistical aggregators.\n \"\"\"\n return self.aggregators.__len__()\n\n def __eq__(self, other):\n \"\"\"\n Check whether two aggregators are equal (just for the purpose of compatibility with the base Mapping class).\n \"\"\"\n if isinstance(other, self.__class__):\n # Check statistics, formatting and aggregators.\n return self.statistics == other.statistics and self.formatting == other.formatting and self.aggregators == other.aggregators\n else:\n return False\n\n def __iter__(self):\n \"\"\"\n Return an iterator on the currently tracked statistical aggregators.\n \"\"\"\n return self.aggregators.__iter__()\n\n\n def initialize_csv_file(self, log_dir, filename):\n \"\"\"\n This method creates a new `csv` file and initializes it with a header produced \\\n on the base of the statistical aggregators names.\n\n :param log_dir: Path to file.\n :type log_dir: str\n\n :param filename: Filename to be created.\n :type filename: str\n\n :return: File stream opened for writing.\n\n \"\"\"\n return super().base_initialize_csv_file(log_dir, filename, self.aggregators.keys())\n\n\n def export_to_csv(self, csv_file=None):\n \"\"\"\n This method writes the current statistical aggregators values to the `csv_file` using the associated formatting.\n\n :param csv_file: File stream opened for writing, optional.\n\n \"\"\"\n # Try to use the remembered one. \n if csv_file is None:\n csv_file = self.csv_file\n # If it is still None - well, we cannot do anything more.\n if csv_file is None:\n return\n\n values_str = ''\n\n # Iterate through values and concatenate them.\n for key, value in self.aggregators.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n\n # Add value to string using formatting.\n values_str += format_str.format(value) + \",\"\n\n # Remove last coma.\n if len(values_str) > 1:\n values_str = values_str[:-1]\n # Add last \\n.\n values_str = values_str + '\\n'\n\n csv_file.write(values_str)\n\n def export_to_checkpoint(self):\n \"\"\"\n This method exports the aggregated data into a dictionary using the associated formatting.\n\n \"\"\"\n chkpt = {}\n\n # Iterate through key, values and format them.\n for key, value in self.aggregators.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n\n # Add to dict.\n chkpt[key] = format_str.format(value)\n\n return chkpt \n\n def export_to_string(self, additional_tag=''):\n \"\"\"\n This method returns the current statistical aggregators values in the form of a string using the \\\n associated formatting.\n\n :param additional_tag: An additional tag to append at the end of the created string.\n :type additional_tag: str\n\n\n :return: String being the concatenation of the statistical aggregators names & values.\n\n \"\"\"\n stat_str = ''\n\n # Iterate through keys and values and concatenate them.\n for key, value in self.aggregators.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n stat_str += key + ' '\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n # Add value to string using formatting.\n stat_str += format_str.format(value) + \"; \"\n\n # Remove last two elements.\n if len(stat_str) > 2:\n stat_str = stat_str[:-2]\n \n # Add addtional tag.\n stat_str = stat_str + \" \" + additional_tag\n\n return stat_str\n\n def export_to_tensorboard(self, tb_writer = None):\n \"\"\"\n Method exports current statistical aggregators values to TensorBoard.\n\n :param tb_writer: TensorBoard writer, optional\n :type tb_writer: :py:class:`tensorboardX.SummaryWriter`\n\n \"\"\"\n # Get episode number.\n episode = self.aggregators['episode']\n\n if tb_writer is None:\n tb_writer = self.tb_writer\n # If it is still None - well, we cannot do anything more.\n if tb_writer is None:\n return\n\n # Iterate through keys and values and concatenate them.\n for key, value in self.aggregators.items():\n # Skip episode.\n if key == 'episode':\n continue\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n tb_writer.add_scalar(key, value, episode)\n" }, { "path": "ptp/utils/statistics_collector.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta & Vincent Marois\"\n\nfrom collections.abc import Mapping\n\n\nclass StatisticsCollector(Mapping):\n \"\"\"\n Specialized class used for the collection and export of statistics during\\\n training, validation and testing.\n\n Inherits :py:class:`collections.Mapping`, therefore it offers functionality close to a ``dict``.\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Initialization - creates dictionaries for statistics and formatting.\n \"\"\"\n super(StatisticsCollector, self).__init__()\n\n # Set default \"output streams\" for none.\n self.tb_writer = None\n self.csv_file = None\n\n self.statistics = dict()\n self.formatting = dict()\n\n def add_statistics(self, key, formatting):\n \"\"\"\n Add a statistics to collector.\n The value of associated to the key is of type ``list``.\n\n :param key: Key of the statistics.\n :type key: str\n\n :param formatting: Formatting that will be used when logging and exporting to CSV.\n\n \"\"\"\n self.formatting[key] = formatting\n\n # instantiate associated value as a list.\n self.statistics[key] = list()\n\n def __getitem__(self, key):\n \"\"\"\n Get statistics value for given key.\n\n :param key: Key to value in parameters.\n :type key: str\n\n :return: Statistics value list associated with given key.\n\n \"\"\"\n return self.statistics[key]\n\n def __setitem__(self, key, value):\n \"\"\"\n Add value to the list of the statistic associated with a given key.\n\n :param key: Key to value in parameters.\n :param value: Statistics value to append to the list associated with given key.\n\n \"\"\"\n self.statistics[key].append(value)\n\n def __delitem__(self, key):\n \"\"\"\n Delete the specified key.\n\n :param key: Key to be deleted.\n\n \"\"\"\n del self.statistics[key]\n\n def __len__(self):\n \"\"\"\n Returns \"length\" of ``self.statistics`` (i.e. number of tracked values).\n \"\"\"\n return self.statistics.__len__()\n\n def __iter__(self):\n \"\"\"\n Iterator.\n \"\"\"\n return self.statistics.__iter__()\n\n def __eq__(self, other):\n \"\"\"\n Check whether two collectors are equal (just for the purpose of compatibility with the base Mapping class).\n \"\"\"\n if isinstance(other, self.__class__):\n # Check statistics and formatting.\n return self.statistics == other.statistics and self.formatting == other.formatting\n else:\n return False\n\n def empty(self):\n \"\"\"\n Empty the list associated to the keys of the current statistics collector.\n\n \"\"\"\n for key in self.statistics.keys():\n del self.statistics[key][:]\n\n\n def base_initialize_csv_file(self, log_dir, filename, keys):\n \"\"\"\n This method creates a new `csv` file and initializes it with a header produced \\\n on the base of the statistical aggregators names.\n\n :param log_dir: Path to file.\n :type log_dir: str\n\n :param filename: Filename to be created.\n :type filename: str\n\n :param keys: Names of keys that will be used as header of columns in csv file.\n\n :return: File stream opened for writing.\n\n \"\"\"\n header_str = ''\n\n # Iterate through keys and concatenate them.\n for key in keys:\n # If formatting is set to '' - ignore this key.\n if self.formatting.get(key) is not None:\n header_str += key + \",\"\n\n # Remove last coma.\n if len(header_str) > 0:\n header_str = header_str[:-1]\n # Add \\n.\n header_str = header_str + '\\n'\n\n # Open file for writing.\n self.csv_file = open(log_dir + filename, 'w', 1)\n self.csv_file.write(header_str)\n\n return self.csv_file \n\n\n def initialize_csv_file(self, log_dir, filename):\n \"\"\"\n This method creates a new `csv` file and initializes it with a header produced \\\n on the base of the statistical aggregators names.\n\n :param log_dir: Path to file.\n :type log_dir: str\n\n :param filename: Filename to be created.\n :type filename: str\n\n :return: File stream opened for writing.\n\n \"\"\"\n return self.base_initialize_csv_file(log_dir, filename, self.statistics.keys())\n\n\n def export_to_csv(self, csv_file=None):\n \"\"\"\n Method writes current statistics to csv using the possessed formatting.\n\n :param csv_file: File stream opened for writing, optional\n\n \"\"\"\n # Try to use the remembered one. \n if csv_file is None:\n csv_file = self.csv_file\n # If it is still None - well, we cannot do anything more.\n if csv_file is None:\n return\n\n # Iterate through values and concatenate them.\n values_str = ''\n for key, value in self.statistics.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n\n # Add value to string using formatting.\n if len(value) > 0:\n values_str += format_str.format(value[-1])\n values_str += \",\"\n\n # Remove last coma.\n if len(values_str) > 1:\n values_str = values_str[:-1]\n # Add last \\n.\n values_str = values_str + '\\n'\n\n csv_file.write(values_str)\n\n def export_to_checkpoint(self):\n \"\"\"\n This method exports the collected data into a dictionary using the associated formatting.\n\n \"\"\"\n chkpt = {}\n\n # Iterate through key, values and format them.\n for key, value in self.statistics.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n\n # Add to dict.\n if len(value) > 0:\n chkpt[key] = format_str.format(value[-1])\n\n return chkpt\n\n def export_to_string(self, additional_tag=''):\n \"\"\"\n Method returns current statistics in the form of string using the\n possessed formatting.\n\n :param additional_tag: An additional tag to append at the end of the created string.\n :type additional_tag: str\n\n\n :return: String being the concatenation of the statistics names & values.\n\n \"\"\"\n # Iterate through keys and values and concatenate them.\n stat_str = ''\n for key, value in self.statistics.items():\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n stat_str += key + ' '\n # Get formatting - using '{}' as default.\n format_str = self.formatting.get(key, '{}')\n # Add value to string using formatting.\n if len(value) > 0:\n stat_str += format_str.format(value[-1])\n stat_str += \"; \"\n\n # Remove last two elements.\n if len(stat_str) > 2:\n stat_str = stat_str[:-2]\n \n # Add addtional tag.\n stat_str = stat_str + \" \" + additional_tag\n\n return stat_str\n\n def initialize_tensorboard(self, tb_writer):\n \"\"\" \n Memorizes the writer that will be used with this collector.\n \"\"\" \n self.tb_writer = tb_writer\n\n def export_to_tensorboard(self, tb_writer=None):\n \"\"\"\n Method exports current statistics to tensorboard.\n\n :param tb_writer: TensorBoard writer, optional.\n :type tb_writer: :py:class:`tensorboardX.SummaryWriter`\n\n \"\"\"\n # Get episode number.\n episode = self.statistics['episode'][-1]\n\n if tb_writer is None:\n tb_writer = self.tb_writer\n # If it is still None - well, we cannot do anything more.\n if tb_writer is None:\n return\n\n # Iterate through keys and values and concatenate them.\n for key, value in self.statistics.items():\n # Skip episode.\n if key == 'episode':\n continue\n # If formatting is set to None - ignore this key.\n if self.formatting.get(key) is not None:\n tb_writer.add_scalar(key, value[-1], episode)\n" }, { "path": "ptp/utils/termination_condition.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM tkornuta, Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\n\nclass TerminationCondition(Exception):\n \"\"\" Exception thrown when worker is supposed to terminate its operation. \"\"\"\n def __init__(self, msg):\n \"\"\" Stores message \"\"\"\n self.msg = msg\n\n def __str__(self):\n \"\"\" Prints the message \"\"\"\n return repr(self.msg)\n\n" }, { "path": "ptp/workers/__init__.py", "content": "from .worker import Worker\nfrom .trainer import Trainer\n#from .offline_trainer import OfflineTrainer\nfrom .online_trainer import OnlineTrainer\nfrom .processor import Processor\n\n__all__ = [\n 'Worker',\n 'Trainer',\n #'OfflineTrainer',\n 'OnlineTrainer',\n 'Processor'\n ]\n" }, { "path": "ptp/workers/offline_trainer.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta, Vincent Marois\"\n\nimport torch\nimport numpy as np\n\nfrom ptp.workers.trainer import Trainer\nimport ptp.configuration.config_parsing as config_parsing\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.utils.termination_condition import TerminationCondition\n\nclass OfflineTrainer(Trainer):\n \"\"\"\n Implementation for the epoch-based ``OfflineTrainer``.\n\n ..note::\n\n The default ``OfflineTrainer`` is based on epochs. \\\n An epoch is defined as passing through all samples of a finite-size dataset.\\\n The ``OfflineTrainer`` allows to loop over all samples from the training set many times i.e. in many epochs. \\\n When an epochs finishes, it performs a similar step for the validation set and collects the statistics.\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Constructor. It on calls the ``Trainer`` constructor as the initialization phase is identical to the one from ``Trainer``.\n \"\"\" \n # Call base constructor to set up app state, registry and add default config.\n super(OfflineTrainer, self).__init__(\"OfflineTrainer\", OfflineTrainer)\n\n def setup_experiment(self):\n \"\"\"\n Sets up experiment for episode trainer:\n\n - Calls base class setup_experiment to parse the command line arguments,\n - Sets up the terminal conditions (loss threshold, episodes & epochs (optional) limits).\n\n \"\"\"\n # Call base method to parse all command line arguments, load configuration, create tasks and model etc.\n super(OfflineTrainer, self).setup_experiment()\n\n # In this trainer Partial Validation is mandatory, hence interval must be > 0.\n self.partial_validation_interval = self.config['validation']['partial_validation_interval']\n if self.partial_validation_interval <= 0:\n self.logger.info(\"Partial Validation deactivated\")\n else:\n self.logger.info(\"Partial Validation activated with interval equal to {} episodes\".format(self.partial_validation_interval))\n\n ################# TERMINAL CONDITIONS ################# \n log_str = 'Terminal conditions:\\n' + '='*80 + \"\\n\"\n\n # Terminal condition I: loss. \n self.loss_stop_threshold = self.config_training['terminal_conditions']['loss_stop_threshold']\n log_str += \" I: Setting Loss Stop Threshold to {}\\n\".format(self.loss_stop_threshold)\n\n # Terminal condition II: early stopping. \n self.early_stop_validations = self.config_training['terminal_conditions']['early_stop_validations']\n if self.early_stop_validations <= 0:\n log_str += \" II: Termination based on Early Stopping is disabled\\n\"\n # Set to infinity.\n self.early_stop_validations = np.Inf\n else:\n log_str += \" II: Setting the Number of Validations in Early Stopping to: {}\\n\".format(self.early_stop_validations)\n\n # Terminal condition III: max epochs. Mandatory.\n self.epoch_limit = self.config_training[\"terminal_conditions\"][\"epoch_limit\"]\n if self.epoch_limit <= 0:\n self.logger.error(\"OffLine Trainer relies on epochs, thus Epoch Limit must be a positive number!\")\n exit(-5)\n else:\n log_str += \" III: Setting the Epoch Limit to: {}\\n\".format(self.epoch_limit)\n\n # Log the epoch size in terms of episodes.\n self.epoch_size = self.training.get_epoch_size()\n log_str += \" Epoch size in terms of training episodes: {}\\n\".format(self.epoch_size)\n\n # Terminal condition IV: max episodes. Optional.\n self.episode_limit = self.config_training['terminal_conditions']['episode_limit']\n if self.episode_limit < 0:\n log_str += \" IV: Termination based on Episode Limit is disabled\\n\"\n # Set to infinity.\n self.episode_limit = np.Inf\n else:\n log_str += \" IV: Setting the Episode Limit to: {}\\n\".format(self.episode_limit)\n\n # Ok, finally print it.\n log_str += '='*80 \n self.logger.info(log_str)\n\n # Export and log configuration, optionally asking the user for confirmation.\n config_parsing.display_parsing_results(self.logger, self.app_state.args, self.unparsed)\n config_parsing.display_globals(self.logger, self.app_state.globalitems())\n config_parsing.export_experiment_configuration_to_yml(self.logger, self.app_state.log_dir, \"training_configuration.yml\", self.config, self.app_state.args.confirm)\n\n def run_experiment(self):\n \"\"\"\n Main function of the ``OfflineTrainer``, runs the experiment.\n\n Iterates over the (cycled) DataLoader (one iteration = one episode).\n\n .. note::\n\n The test for terminal conditions (e.g. convergence) is done at the end of each episode. \\\n The terminal conditions are as follows:\n\n - I. The loss is below the specified threshold (using the partial validation loss),\n - II. Early stopping is set and the full validation loss did went down \\\n for the indicated number of validation steps,\n - III. The maximum number of episodes has been met (OPTIONAL),\n - IV. The maximum number of epochs has been met.\n \n Additionally, experiment can be stopped by the user by pressing 'Stop experiment' \\\n during visualization.\n\n\n The function does the following for each episode:\n\n - Handles curriculum learning if set,\n - Resets the gradients\n - Forwards pass of the model,\n - Logs statistics and exports to TensorBoard (if set),\n - Computes gradients and update weights\n - Activate visualization if set,\n - Validate the model on a batch according to the validation frequency.\n - Checks the above terminal conditions.\n\n\n \"\"\"\n # Initialize TensorBoard and statistics collection.\n self.initialize_statistics_collection()\n self.initialize_tensorboard()\n\n try:\n '''\n Main training and validation loop.\n '''\n # Reset the counters.\n self.app_state.episode = -1\n self.app_state.epoch = -1\n # Set initial status.\n training_status = \"Not Converged\"\n\n ################################################################################################\n # Beginning of external \"epic loop\".\n ################################################################################################\n while(True):\n self.app_state.epoch += 1\n self.logger.info('Starting next epoch: {}\\n{}'.format(self.app_state.epoch, '='*80))\n\n # Inform the task managers that epoch has started.\n self.training.initialize_epoch()\n self.validation.initialize_epoch()\n\n # Apply curriculum learning - change Task parameters.\n self.curric_done = self.training.task.curriculum_learning_update_params(\n 0 if self.app_state.episode < 0 else self.app_state.episode,\n self.app_state.epoch)\n \n\n # Empty the statistics collector.\n self.training_stat_col.empty()\n \n ############################################################################################\n # Beginning of internal \"episodic loop\".\n ############################################################################################\n for training_batch in self.training.dataloader:\n # Next episode.\n self.app_state.episode += 1\n\n # reset all gradients\n self.optimizer.zero_grad()\n\n # Turn on training mode for the model.\n self.pipeline.train()\n\n # 1. Perform forward step.\n self.pipeline.forward(training_batch)\n\n # 2. Calculate statistics.\n self.collect_all_statistics(self.training, self.pipeline, training_batch, self.training_stat_col)\n\n # 3. Backward gradient flow.\n self.pipeline.backward(training_batch)\n\n # Check the presence of the 'gradient_clipping' parameter.\n try:\n # if present - clip gradients to a range (-gradient_clipping, gradient_clipping)\n val = self.config_training['gradient_clipping']\n torch.nn.utils.clip_grad_value_(self.pipeline.parameters(), val)\n except KeyError:\n # Else - do nothing.\n pass\n\n # 4. Perform optimization.\n self.optimizer.step()\n\n # 5. Log collected statistics.\n # 5.1. Export to csv - at every step.\n self.training_stat_col.export_to_csv()\n\n # 5.2. Export data to TensorBoard - at logging frequency.\n if (self.training_batch_writer is not None) and \\\n (self.app_state.episode % self.app_state.args.logging_interval == 0):\n self.training_stat_col.export_to_tensorboard()\n\n # Export histograms.\n if self.app_state.args.tensorboard >= 1:\n for name, param in self.pipeline.named_parameters():\n try:\n self.training_batch_writer.add_histogram(name, \n param.data.cpu().numpy(), self.app_state.episode, bins='doane')\n\n except Exception as e:\n self.logger.error(\" {} :: data :: {}\".format(name, e))\n\n # Export gradients.\n if self.app_state.args.tensorboard >= 2:\n for name, param in self.pipeline.named_parameters():\n try:\n self.training_batch_writer.add_histogram(name + '/grad', \n param.grad.data.cpu().numpy(), self.app_state.episode, bins='doane')\n\n except Exception as e:\n self.logger.error(\" {} :: grad :: {}\".format(name, e))\n\n # 5.3. Log to logger - at logging frequency.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n self.logger.info(self.training_stat_col.export_to_string())\n\n # 6. Validate and (optionally) save the model.\n if self.partial_validation_interval > 0 and (self.app_state.episode % self.partial_validation_interval) == 0:\n # Clear the validation batch from all items aside of the ones originally returned by the task.\n self.validation.batch.reinitialize(self.validation.task.output_data_definitions())\n # Perform validation.\n self.validate_on_batch(self.validation.batch)\n # Do not save the model: OfflineTrainer uses the full set to determine whether to save or not.\n\n # III. The episodes number limit has been reached.\n if self.app_state.episode+1 >= self.episode_limit: # = np.Inf when inactive.\n # If we reach this condition, then it is possible that the model didn't converge correctly\n # but it currently might get better since last validation.\n training_status = \"Not converged: Episode Limit reached\"\n raise TerminationCondition(training_status)\n \n ############################################################################################\n # End of internal \"episodic loop\".\n ############################################################################################\n\n # Epoch just ended!\n self.logger.info('End of epoch: {}\\n{}'.format(self.app_state.epoch, '='*80))\n\n # Aggregate training statistics for the epoch.\n self.aggregate_all_statistics(self.training, self.pipeline, self.training_stat_col, self.training_stat_agg)\n self.export_all_statistics( self.training_stat_agg, '[Full Training]')\n\n # Inform the training task manager that the epoch has ended.\n self.training.finalize_epoch()\n\n # Validate over the entire validation set.\n self.validate_on_set()\n\n # Get loss.\n validation_set_loss = self.pipeline.return_loss_on_set(self.validation_stat_agg)\n\n # Save the pipeline using the latest validation statistics.\n self.pipeline.save(self.checkpoint_dir, training_status, validation_set_loss)\n\n # Inform the validation task manager that the epoch has ended.\n self.validation.finalize_epoch()\n\n # Terminal conditions.\n # I. The loss is < threshold (only when curriculum learning is finished if set).\n # We check that condition only in validation step!\n if self.curric_done or not self.must_finish_curriculum:\n\n # Check the Partial Validation loss.\n if (validation_set_loss < self.loss_stop_threshold):\n # Change the status.\n training_status = \"Converged (Full Validation Loss went below \" \\\n \"Loss Stop threshold of {})\".format(self.loss_stop_threshold)\n\n # Save the pipeline (update its statistics).\n self.pipeline.save(self.checkpoint_dir, training_status, validation_set_loss)\n # And leave both loops.\n raise TerminationCondition(training_status)\n\n # II. Early stopping is set and loss hasn't improved by delta in n epochs.\n if self.pipeline.validation_loss_down_counter >= self.early_stop_validations:\n training_status = \"Not converged: reached limit of validations without improvement (Early Stopping)\"\n raise TerminationCondition(training_status)\n\n # IV. Epoch limit has been reached.\n if self.app_state.epoch+1 >= self.epoch_limit:\n training_status = \"Not converged: Epoch Limit reached\"\n # \"Finish\" the training.\n raise TerminationCondition(training_status)\n\n ################################################################################################\n # End of external \"epic loop\".\n ################################################################################################\n\n except TerminationCondition as e:\n # End of main training and validation loop. Perform final full validation.\n\n self.logger.info('\\n' + '='*80)\n self.logger.info('Training finished because {}'.format(training_status))\n\n # If episode limit was reached - perform last validation on the full set.\n if training_status == \"Not converged: Episode Limit reached\":\n # Validate over the entire validation set.\n self.validate_on_set()\n # Get loss.\n validation_set_loss = self.pipeline.return_loss_on_set(self.validation_stat_agg)\n # Save the pipeline using the latest validation statistics.\n self.pipeline.save(self.checkpoint_dir, training_status, validation_set_loss)\n\n\n self.logger.info('Experiment finished!')\n\n except SystemExit as e:\n # the training did not end properly\n self.logger.error('Experiment interrupted because {}'.format(e))\n except ConfigurationError as e:\n # the training did not end properly\n self.logger.error('Experiment interrupted because {}'.format(e))\n except KeyboardInterrupt:\n # the training did not end properly\n self.logger.error('Experiment interrupted!')\n finally:\n # Finalize statistics collection.\n self.finalize_statistics_collection()\n self.finalize_tensorboard()\n self.logger.info(\"Experiment logged to: {}\".format(self.app_state.log_dir))\n\n\ndef main():\n \"\"\"\n Entry point function for the ``OfflineTrainer``.\n \"\"\"\n # Create trainer.\n trainer = OfflineTrainer()\n # Parse args, load configuration and create all required objects.\n trainer.setup_experiment()\n # GO!\n trainer.run_experiment()\n\nif __name__ == '__main__':\n main()" }, { "path": "ptp/workers/online_trainer.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta, Vincent Marois\"\n\nimport torch\nimport numpy as np\n\nfrom ptp.workers.trainer import Trainer\nimport ptp.configuration.config_parsing as config_parsing\nfrom ptp.configuration.configuration_error import ConfigurationError\nfrom ptp.utils.termination_condition import TerminationCondition\n\nclass OnlineTrainer(Trainer):\n \"\"\"\n Implementation for the episode-based ``OnlineTrainer``.\n\n ..note ::\n\n The ``OfflineTrainer`` is based on epochs. While an epoch can be defined for all finite-size datasets, \\\n it makes less sense for tasks which have a very large, almost infinite, dataset (like algorithmic \\\n tasks, which generate random data on-the-fly). \\\n \n This is why this OnlineTrainer was implemented. Despite the fact it has the notion of epoch, it is more \\\n flexible and operates on episodes (we call an iteration on a single batch an episode). \\\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Constructor. It on calls the ``Trainer`` constructor as the initialization phase is identical to the one from ``Trainer``.\n \"\"\" \n # Call base constructor to set up app state, registry and add default config.\n super(OnlineTrainer, self).__init__(\"OnlineTrainer\", OnlineTrainer)\n\n def setup_experiment(self):\n \"\"\"\n Sets up experiment for episode trainer:\n\n - Calls base class setup_experiment to parse the command line arguments,\n - Sets up the terminal conditions (loss threshold, episodes & epochs (optional) limits).\n\n \"\"\"\n # Call base method to parse all command line arguments, load configuration, create tasks and model etc.\n super(OnlineTrainer, self).setup_experiment()\n\n # In this trainer Partial Validation is mandatory, hence interval must be > 0.\n self.partial_validation_interval = self.config['validation']['partial_validation_interval']\n if self.partial_validation_interval <= 0:\n self.logger.error(\"Online Trainer relies on Partial Validation, thus 'partial_validation_interval' must be a positive number!\")\n exit(-4)\n else:\n self.logger.info(\"Partial Validation activated with interval equal to {} episodes\\n\".format(self.partial_validation_interval))\n\n ################# TERMINAL CONDITIONS ################# \n log_str = 'Terminal conditions:\\n' + '='*80 + \"\\n\"\n\n # Terminal condition I: loss. \n self.loss_stop_threshold = self.config_training['terminal_conditions']['loss_stop_threshold']\n log_str += \" I: Setting Loss Stop Threshold to {}\\n\".format(self.loss_stop_threshold)\n\n # Terminal condition II: early stopping. \n self.early_stop_validations = self.config_training['terminal_conditions']['early_stop_validations']\n if self.early_stop_validations <= 0:\n log_str += \" II: Termination based on Early Stopping is disabled\\n\"\n # Set to infinity.\n self.early_stop_validations = np.Inf\n else:\n log_str += \" II: Setting the Number of Validations in Early Stopping to: {}\\n\".format(self.early_stop_validations)\n\n # Terminal condition III: max epochs (Optional for this trainer)\n self.epoch_limit = self.config_training[\"terminal_conditions\"][\"epoch_limit\"]\n if self.epoch_limit <= 0:\n log_str += \" III: Termination based on Epoch Limit is disabled\\n\"\n # Set to infinity.\n self.epoch_limit = np.Inf\n else:\n log_str += \" III: Setting the Epoch Limit to: {}\\n\".format(self.epoch_limit)\n\n # Log the epoch size in terms of episodes.\n self.epoch_size = self.training.get_epoch_size()\n log_str += \" Epoch size in terms of training episodes: {}\\n\".format(self.epoch_size)\n\n # Terminal condition IV: max episodes. Mandatory.\n self.episode_limit = self.config_training['terminal_conditions']['episode_limit']\n if self.episode_limit <= 0:\n self.logger.error(\"OnLine Trainer relies on episodes, thus 'episode_limit' must be a positive number!\")\n exit(-5)\n else:\n log_str += \" IV: Setting the Episode Limit to: {}\\n\".format(self.episode_limit)\n # Ok, finally print it.\n log_str += '='*80 \n self.logger.info(log_str)\n\n # Export and log configuration, optionally asking the user for confirmation.\n config_parsing.display_parsing_results(self.logger, self.app_state.args, self.unparsed)\n config_parsing.display_globals(self.logger, self.app_state.globalitems())\n config_parsing.export_experiment_configuration_to_yml(self.logger, self.app_state.log_dir, \"training_configuration.yml\", self.config, self.app_state.args.confirm)\n\n def run_experiment(self):\n \"\"\"\n Main function of the ``OnlineTrainer``, runs the experiment.\n\n Iterates over the (cycled) DataLoader (one iteration = one episode).\n\n .. note::\n\n The test for terminal conditions (e.g. convergence) is done at the end of each episode. \\\n The terminal conditions are as follows:\n\n - I. The loss is below the specified threshold (using the partial validation loss),\n - II. Early stopping is set and the full validation loss did went down \\\n for the indicated number of validation steps,\n - III. The maximum number of episodes has been met,\n - IV. The maximum number of epochs has been met (OPTIONAL).\n \n Additionally, experiment can be stopped by the user by pressing 'Stop experiment' \\\n during visualization.\n\n\n The function does the following for each episode:\n\n - Handles curriculum learning if set,\n - Resets the gradients\n - Forwards pass of the model,\n - Logs statistics and exports to TensorBoard (if set),\n - Computes gradients and update weights\n - Activate visualization if set,\n - Validate the model on a batch according to the validation frequency.\n - Checks the above terminal conditions.\n\n\n \"\"\"\n # Initialize TensorBoard and statistics collection.\n self.initialize_statistics_collection()\n self.initialize_tensorboard()\n\n try:\n '''\n Main training and validation loop.\n '''\n # Reset the counters.\n self.app_state.episode = -1\n self.app_state.epoch = -1\n # Set initial status.\n training_status = \"Not Converged\"\n\n ################################################################################################\n # Beginning of external \"epic loop\".\n ################################################################################################\n while(True):\n self.app_state.epoch += 1\n self.logger.info('Starting next epoch: {}\\n{}'.format(self.app_state.epoch, '='*80))\n\n # Inform the task managers that epoch has started.\n self.training.initialize_epoch()\n self.validation.initialize_epoch()\n\n # Apply curriculum learning - change Task parameters.\n self.curric_done = self.training.task.curriculum_learning_update_params(\n 0 if self.app_state.episode < 0 else self.app_state.episode,\n self.app_state.epoch)\n \n\n # Empty the statistics collector.\n self.training_stat_col.empty()\n \n ############################################################################################\n # Beginning of internal \"episodic loop\".\n ############################################################################################\n for training_batch in self.training.dataloader:\n # Next episode.\n self.app_state.episode += 1\n\n # reset all gradients\n self.optimizer.zero_grad()\n\n # Turn on training mode for the model.\n self.pipeline.train()\n\n # 1. Perform forward step.\n self.pipeline.forward(training_batch)\n\n # 2. Calculate statistics.\n self.collect_all_statistics(self.training, self.pipeline, training_batch, self.training_stat_col)\n\n # 3. Backward gradient flow.\n self.pipeline.backward(training_batch)\n\n # Check the presence of the 'gradient_clipping' parameter.\n try:\n # if present - clip gradients to a range (-gradient_clipping, gradient_clipping)\n val = self.config_training['gradient_clipping']\n torch.nn.utils.clip_grad_value_(self.pipeline.parameters(), val)\n except KeyError:\n # Else - do nothing.\n pass\n\n # 4. Perform optimization.\n self.optimizer.step()\n\n # 5. Log collected statistics.\n # 5.1. Export to csv - at every step.\n self.training_stat_col.export_to_csv()\n\n # 5.2. Export data to TensorBoard - at logging frequency.\n if (self.training_batch_writer is not None) and \\\n (self.app_state.episode % self.app_state.args.logging_interval == 0):\n self.training_stat_col.export_to_tensorboard()\n\n # Export histograms.\n if self.app_state.args.tensorboard >= 1:\n for name, param in self.pipeline.named_parameters():\n try:\n self.training_batch_writer.add_histogram(name, \n param.data.cpu().numpy(), self.app_state.episode, bins='doane')\n\n except Exception as e:\n self.logger.error(\" {} :: data :: {}\".format(name, e))\n\n # Export gradients.\n if self.app_state.args.tensorboard >= 2:\n for name, param in self.pipeline.named_parameters():\n try:\n self.training_batch_writer.add_histogram(name + '/grad', \n param.grad.data.cpu().numpy(), self.app_state.episode, bins='doane')\n\n except Exception as e:\n self.logger.error(\" {} :: grad :: {}\".format(name, e))\n\n # 5.3. Log to logger - at logging frequency.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n self.logger.info(self.training_stat_col.export_to_string())\n\n # 6. Validate and (optionally) save the model.\n if (self.app_state.episode % self.partial_validation_interval) == 0:\n\n # Clear the validation batch from all items aside of the ones originally returned by the task.\n self.validation.batch.reinitialize(self.validation.task.output_data_definitions())\n # Perform validation.\n self.validate_on_batch(self.validation.batch)\n # Get loss.\n validation_batch_loss = self.pipeline.return_loss_on_batch(self.validation_stat_col)\n\n # Save the pipeline using the latest validation statistics.\n self.pipeline.save(self.checkpoint_dir, training_status, validation_batch_loss)\n\n # Terminal conditions.\n # I. The loss is < threshold (only when curriculum learning is finished if set).\n # We check that condition only in validation step!\n if self.curric_done or not self.must_finish_curriculum:\n\n # Check the Partial Validation loss.\n if (validation_batch_loss < self.loss_stop_threshold):\n # Change the status.\n training_status = \"Converged (Partial Validation Loss went below \" \\\n \"Loss Stop threshold {})\".format(self.loss_stop_threshold)\n\n # Save the pipeline (update its statistics).\n self.pipeline.save(self.checkpoint_dir, training_status, validation_batch_loss)\n # And leave both loops.\n raise TerminationCondition(training_status)\n\n # II. Early stopping is set and loss hasn't improved by delta in n epochs.\n if self.pipeline.validation_loss_down_counter >= self.early_stop_validations:\n training_status = \"Not converged: reached limit of validations without improvement (Early Stopping)\"\n raise TerminationCondition(training_status)\n\n # III. The episodes number limit has been reached.\n if self.app_state.episode+1 >= self.episode_limit:\n # If we reach this condition, then it is possible that the model didn't converge correctly\n # but it currently might get better since last validation.\n training_status = \"Not converged: Episode Limit reached\"\n raise TerminationCondition(training_status)\n \n ############################################################################################\n # End of internal \"episodic loop\".\n ############################################################################################\n\n # Epoch just ended!\n self.logger.info('End of epoch: {}\\n{}'.format(self.app_state.epoch, '='*80))\n \n # Inform the task managers that the epoch has ended.\n self.training.finalize_epoch()\n self.validation.finalize_epoch()\n\n # Aggregate training statistics for the epoch.\n self.aggregate_all_statistics(self.training, self.pipeline, self.training_stat_col, self.training_stat_agg)\n self.export_all_statistics( self.training_stat_agg, '[Full Training]')\n\n # IV. Epoch limit has been reached.\n if self.app_state.epoch+1 >= self.epoch_limit: # = np.Inf when inactive.\n training_status = \"Not converged: Epoch Limit reached\"\n # \"Finish\" the training.\n raise TerminationCondition(training_status)\n\n ################################################################################################\n # End of external \"epic loop\".\n ################################################################################################\n\n except TerminationCondition as e:\n # End of main training and validation loop. Perform final full validation.\n # Eventually perform \"last\" validation on batch.\n if self.validation_stat_col[\"episode\"][-1] != self.app_state.episode:\n # We still must validate and try to save the model as it may performed better during this episode.\n\n # Clear the validation batch from all items aside of the ones originally returned by the task.\n self.validation.batch.reinitialize(self.validation.task.output_data_definitions())\n # Perform validation.\n self.validate_on_batch(self.validation.batch)\n # Get loss.\n validation_batch_loss = self.pipeline.return_loss_on_batch(self.validation_stat_col)\n\n # Try to save the model using the latest validation statistics.\n self.pipeline.save(self.checkpoint_dir, training_status, validation_batch_loss)\n\n self.logger.info('\\n' + '='*80)\n self.logger.info('Training finished because {}'.format(training_status))\n\n # Validate over the entire validation set.\n self.validate_on_set()\n # Do not save the model, as we tried it already on \"last\" validation batch.\n\n self.logger.info('Experiment finished!')\n\n except SystemExit as e:\n # the training did not end properly\n self.logger.error('Experiment interrupted because {}'.format(e))\n except ConfigurationError as e:\n # the training did not end properly\n self.logger.error('Experiment interrupted because {}'.format(e))\n except KeyboardInterrupt:\n # the training did not end properly\n self.logger.error('Experiment interrupted!')\n finally:\n # Finalize statistics collection.\n self.finalize_statistics_collection()\n self.finalize_tensorboard()\n self.logger.info(\"Experiment logged to: {}\".format(self.app_state.log_dir))\n\n\ndef main():\n \"\"\"\n Entry point function for the ``OnlineTrainer``.\n \"\"\"\n # Create trainer.\n trainer = OnlineTrainer()\n # Parse args, load configuration and create all required objects.\n trainer.setup_experiment()\n # GO!\n trainer.run_experiment()\n\nif __name__ == '__main__':\n main()" }, { "path": "ptp/workers/processor.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta, Vincent Marois, Younes Bouhadjar\"\n\nfrom os import path,makedirs\nimport torch\nfrom time import sleep\nfrom datetime import datetime\n\nimport ptp.configuration.config_parsing as config_parsing\nimport ptp.utils.logger as logging\n\nfrom ptp.workers.worker import Worker\n\nfrom ptp.application.task_manager import TaskManager\nfrom ptp.application.pipeline_manager import PipelineManager\n\nfrom ptp.utils.statistics_collector import StatisticsCollector\nfrom ptp.utils.statistics_aggregator import StatisticsAggregator\n\n\nclass Processor(Worker):\n \"\"\"\n Defines the basic ``Processor``.\n\n If defining another type of Processor, it should subclass it.\n\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Calls the ``Worker`` constructor, adds some additional arguments to parser.\n \"\"\" \n # Call base constructor to set up app state, registry and add default params.\n super(Processor, self).__init__(\"Processor\", Processor)\n\n self.parser.add_argument(\n '--section',\n dest='section_name',\n type=str,\n default=\"test\",\n help='Name of the section defining the specific set to be processed (DEFAULT: test)')\n\n def setup_global_experiment(self):\n \"\"\"\n Sets up the global test experiment for the ``Processor``:\n\n - Checks that the model to use exists\n\n - Checks that the configuration file exists\n\n - Creates the configuration\n\n The rest of the experiment setup is done in :py:func:`setup_individual_experiment()` \\\n to allow for multiple tests suppport.\n\n \"\"\"\n # Call base method to parse all command line arguments and add default sections.\n super(Processor, self).setup_experiment()\n\n # \"Pass\" configuration parameters from the default_test section to section indicated by the section_name.\n self.config.add_default_params({ self.app_state.args.section_name: self.config['default_test'].to_dict()} )\n self.config.del_default_params('default_test')\n \n # Retrieve checkpoint file.\n chkpt_file = self.app_state.args.load_checkpoint\n\n # Check the presence of the CUDA-compatible devices.\n if self.app_state.args.use_gpu and (torch.cuda.device_count() == 0):\n self.logger.error(\"Cannot use GPU as there are no CUDA-compatible devices present in the system!\")\n exit(-1)\n\n # Config that will be used.\n abs_root_configs = None\n\n # Check if checkpoint file was indicated.\n if chkpt_file != \"\":\n #print('Please pass path to and name of the file containing pipeline to be loaded as --load parameter')\n #exit(-2)\n\n # Check if file with model exists.\n if not path.isfile(chkpt_file):\n print('Checkpoint file {} does not exist'.format(chkpt_file))\n exit(-3)\n\n # Extract path.\n self.abs_path, _ = path.split(path.dirname(path.expanduser(chkpt_file)))\n\n # Use the \"default\" config.\n abs_root_configs = [path.join(self.abs_path, 'training_configuration.yml')]\n\n # Check if config file was indicated by the user.\n if self.app_state.args.config != '':\n # Split and make them absolute.\n root_configs = self.app_state.args.config.replace(\" \", \"\").split(',')\n # If there are - expand them to absolute paths.\n abs_root_configs = [path.expanduser(config) for config in root_configs]\n\n # Using name of the first configuration file from command line.\n basename = path.basename(root_configs[0])\n # Take config filename without extension.\n pipeline_name = path.splitext(basename)[0] \n\n # Use path to experiments + pipeline.\n self.abs_path = path.join(path.expanduser(self.app_state.args.expdir), pipeline_name)\n\n\n if abs_root_configs is None:\n print('Please indicate configuration file to be used (--config) and/or pass path to and name of the file containing pipeline to be loaded (--load)')\n exit(-2)\n\n\n # Get the list of configurations which need to be loaded.\n configs_to_load = config_parsing.recurrent_config_parse(abs_root_configs, [], self.app_state.absolute_config_path)\n\n # Read the YAML files one by one - but in reverse order -> overwrite the first indicated config(s)\n config_parsing.reverse_order_config_load(self.config, configs_to_load)\n\n # -> At this point, the Config Registry contains the configuration loaded (and overwritten) from several files.\n\n def setup_individual_experiment(self):\n \"\"\"\n Setup individual test experiment in the case of multiple tests, or the main experiment in the case of \\\n one test experiment.\n\n - Set up the log directory path\n\n - Set random seeds\n\n - Creates the pipeline consisting of many components\n\n - Creates testing task manager\n\n - Performs testing of compatibility of testing pipeline\n\n \"\"\"\n\n # Get test section.\n try:\n self.tsn = self.app_state.args.section_name \n self.config_test = self.config[self.tsn]\n if self.config_test is None:\n raise KeyError()\n except KeyError:\n print(\"Error: Couldn't retrieve the section '{}' from the loaded configuration\".format(self.tsn))\n exit(-1)\n\n # Get testing task type.\n try:\n _ = self.config_test['task']['type']\n except KeyError:\n print(\"Error: Couldn't retrieve the task 'type' from the '{}' section in the loaded configuration\".format(self.tsn))\n exit(-5)\n\n # Get pipeline section.\n try:\n psn = self.app_state.args.pipeline_section_name\n self.config_pipeline = self.config[psn]\n if self.config_pipeline is None:\n raise KeyError()\n except KeyError:\n print(\"Error: Couldn't retrieve the pipeline section '{}' from the loaded configuration\".format(psn))\n exit(-1)\n\n # Get pipeline name.\n try:\n pipeline_name = self.config_pipeline['name']\n except KeyError:\n print(\"Error: Couldn't retrieve the pipeline 'name' from the loaded configuration\")\n exit(-6)\n \n # Prepare output paths for logging\n while True:\n # Dirty fix: if log_dir already exists, wait for 1 second and try again\n try:\n time_str = self.tsn+'_{0:%Y%m%d_%H%M%S}'.format(datetime.now())\n if self.app_state.args.exptag != '':\n time_str = time_str + \"_\" + self.app_state.args.exptag\n self.app_state.log_dir = self.abs_path + '/' + time_str + '/'\n # Lowercase dir.\n self.app_state.log_dir = self.app_state.log_dir.lower()\n makedirs(self.app_state.log_dir, exist_ok=False)\n except FileExistsError:\n sleep(1)\n else:\n break\n\n # Set log dir.\n self.app_state.log_file = self.app_state.log_dir + 'processor.log'\n # Initialize logger in app state.\n self.app_state.logger = logging.initialize_logger(\"AppState\")\n # Add handlers for the logfile to worker logger.\n logging.add_file_handler_to_logger(self.logger)\n self.logger.info(\"Logger directory set to: {}\".format(self.app_state.log_dir ))\n\n # Set cpu/gpu types.\n self.app_state.set_types()\n\n # Set random seeds in the testing section.\n self.set_random_seeds(self.tsn, self.config_test)\n\n # Total number of detected errors.\n errors =0\n\n ################# TESTING PROBLEM ################# \n\n # Build the used task manager.\n self.pm = TaskManager(self.tsn, self.config_test) \n errors += self.pm.build()\n\n\n # check if the maximum number of episodes is specified, if not put a\n # default equal to the size of the dataset (divided by the batch size)\n # So that by default, we loop over the test set once.\n task_size_in_episodes = len(self.pm)\n\n if self.config_test[\"terminal_conditions\"][\"episode_limit\"] == -1:\n # Overwrite the config value!\n self.config_test['terminal_conditions'].add_config_params({'episode_limit': task_size_in_episodes})\n\n # Warn if indicated number of episodes is larger than an epoch size:\n if self.config_test[\"terminal_conditions\"][\"episode_limit\"] > task_size_in_episodes:\n self.logger.warning('Indicated limit of number of episodes is larger than one epoch, reducing it.')\n # Overwrite the config value!\n self.config_test['terminal_conditions'].add_config_params({'episode_limit': task_size_in_episodes})\n\n self.logger.info(\"Limiting the number of episodes to: {}\".format(\n self.config_test[\"terminal_conditions\"][\"episode_limit\"]))\n\n ###################### PIPELINE ######################\n \n # Build the pipeline using the loaded configuration and global variables.\n self.pipeline = PipelineManager(pipeline_name, self.config_pipeline)\n errors += self.pipeline.build()\n\n # Show pipeline.\n summary_str = self.pipeline.summarize_all_components_header()\n summary_str += self.pm.task.summarize_io(self.tsn)\n summary_str += self.pipeline.summarize_all_components()\n self.logger.info(summary_str)\n\n # Check errors.\n if errors > 0:\n self.logger.error('Found {} errors, terminating execution'.format(errors))\n exit(-7)\n\n # Handshake definitions.\n self.logger.info(\"Handshaking testing pipeline\")\n defs_testing = self.pm.task.output_data_definitions()\n errors += self.pipeline.handshake(defs_testing)\n\n # Check errors.\n if errors > 0:\n self.logger.error('Found {} errors, terminating execution'.format(errors))\n exit(-2)\n\n # Check if there are any models in the pipeline.\n if len(self.pipeline.models) == 0:\n self.logger.error('Cannot proceed with training, as there are no trainable models in the pipeline')\n exit(-3)\n\n\n # Load the pretrained models params from checkpoint.\n try: \n # Check command line arguments, then check load option in config.\n if self.app_state.args.load_checkpoint != \"\":\n pipeline_name = self.app_state.args.load_checkpoint\n msg = \"command line (--load)\"\n elif \"load\" in self.config_pipeline:\n pipeline_name = self.config_pipeline['load']\n msg = \"'pipeline' section of the configuration file\"\n else:\n pipeline_name = \"\"\n # Try to load the the whole pipeline.\n if pipeline_name != \"\":\n if path.isfile(pipeline_name):\n # Load parameters from checkpoint.\n self.pipeline.load(pipeline_name)\n else:\n raise Exception(\"Couldn't load the checkpoint {} indicated in the {}: file does not exist\".format(pipeline_name, msg))\n # If we succeeded, we do not want to load the models from the file anymore!\n else:\n # Try to load the models parameters - one by one, if set so in the configuration file.\n self.pipeline.load_models()\n \n except KeyError:\n self.logger.error(\"File {} indicated in the {} seems not to be a valid model checkpoint\".format(pipeline_name, msg))\n exit(-5)\n except Exception as e:\n self.logger.error(e)\n # Exit by following the logic: if user wanted to load the model but failed, then continuing the experiment makes no sense.\n exit(-6)\n\n\n # Log the model summaries.\n summary_str = self.pipeline.summarize_models_header()\n summary_str += self.pipeline.summarize_models()\n self.logger.info(summary_str)\n\n # Move the models in the pipeline to GPU.\n if self.app_state.args.use_gpu:\n self.pipeline.cuda()\n\n # Turn on evaluation mode.\n self.pipeline.eval()\n\n # Export and log configuration, optionally asking the user for confirmation.\n config_parsing.display_parsing_results(self.logger, self.app_state.args, self.unparsed)\n config_parsing.display_globals(self.logger, self.app_state.globalitems())\n config_parsing.export_experiment_configuration_to_yml(self.logger, self.app_state.log_dir, \"training_configuration.yml\", self.config, self.app_state.args.confirm)\n\n def initialize_statistics_collection(self):\n \"\"\"\n Function initializes all statistics collectors and aggregators used by a given worker,\n creates output files etc.\n \"\"\"\n # Create statistics collector.\n self.stat_col = StatisticsCollector()\n self.add_statistics(self.stat_col)\n self.pm.task.add_statistics(self.stat_col)\n self.pipeline.add_statistics(self.stat_col)\n # Create the csv file to store the statistics.\n self.pm_batch_stats_file = self.stat_col.initialize_csv_file(self.app_state.log_dir, self.tsn+'_statistics.csv')\n\n # Create statistics aggregator.\n self.stat_agg = StatisticsAggregator()\n self.add_aggregators(self.stat_agg)\n self.pm.task.add_aggregators(self.stat_agg)\n self.pipeline.add_aggregators(self.stat_agg)\n # Create the csv file to store the statistic aggregations.\n # Will contain a single row with aggregated statistics.\n self.pm_set_stats_file = self.stat_agg.initialize_csv_file(self.app_state.log_dir, self.tsn+'_set_agg_statistics.csv')\n\n def finalize_statistics_collection(self):\n \"\"\"\n Finalizes statistics collection, closes all files etc.\n \"\"\"\n # Close all files.\n self.pm_batch_stats_file.close()\n self.pm_set_stats_file.close()\n\n def run_experiment(self):\n \"\"\"\n Main function of the ``Processor``: Test the loaded model over the set.\n\n Iterates over the ``DataLoader`` for a maximum number of episodes equal to the set size.\n\n The function does the following for each episode:\n\n - Forwards pass of the model,\n - Logs statistics & accumulates loss,\n - Activate visualization if set.\n\n \"\"\"\n # Initialize tensorboard and statistics collection.\n self.initialize_statistics_collection()\n\n num_samples = len(self.pm)\n\n self.logger.info('Processing the entire set ({} samples in {} episodes)'.format(\n num_samples, len(self.pm.dataloader)))\n\n try:\n # Run in no_grad mode.\n with torch.no_grad():\n # Reset the counter.\n self.app_state.episode = -1\n\n # Inform the task manager that epoch has started.\n self.pm.initialize_epoch()\n\n for batch in self.pm.dataloader:\n # Increment counter.\n self.app_state.episode += 1\n # Terminal condition 0: max test episodes reached.\n if self.app_state.episode == self.config_test[\"terminal_conditions\"][\"episode_limit\"]:\n break\n\n # Forward pass.\n self.pipeline.forward(batch)\n # Collect the statistics.\n self.collect_all_statistics(self.pm, self.pipeline, batch, self.stat_col)\n\n # Export to csv - at every step.\n self.stat_col.export_to_csv()\n\n # Log to logger - at logging frequency.\n if self.app_state.episode % self.app_state.args.logging_interval == 0:\n self.logger.info(self.stat_col.export_to_string('[Partial]'))\n\n # move to next episode.\n self.app_state.episode += 1\n\n # End for.\n # Inform the task managers that the epoch has ended.\n self.pm.finalize_epoch()\n\n self.logger.info('\\n' + '='*80)\n self.logger.info('Processing finished')\n\n # Aggregate statistics for the whole set.\n self.aggregate_all_statistics(self.pm, self.pipeline, self.stat_col, self.stat_agg)\n\n # Export aggregated statistics.\n self.export_all_statistics(self.stat_agg, '[Full Set]')\n\n\n except SystemExit as e:\n # the training did not end properly\n self.logger.error('Experiment interrupted because {}'.format(e))\n except KeyboardInterrupt:\n # the training did not end properly\n self.logger.error('Experiment interrupted!')\n finally:\n # Finalize statistics collection.\n self.finalize_statistics_collection()\n self.logger.info(\"Experiment logged to: {}\".format(self.app_state.log_dir))\n \n\ndef main():\n \"\"\"\n Entry point function for the ``Processor``.\n\n \"\"\"\n processor = Processor()\n # parse args, load configuration and create all required objects.\n processor.setup_global_experiment()\n # finalize the experiment setup\n processor.setup_individual_experiment()\n # run the experiment\n processor.run_experiment()\n\nif __name__ == '__main__':\n main()\n" }, { "path": "ptp/workers/test_data_dict_parallel.py", "content": "\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nimport time\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.components.models.model import Model\n\nfrom ptp.data_types.data_streams import DataStreams\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.utils.data_streams_parallel import DataStreamsParallel\n\n\nclass RandomDataset(Task):\n\n def __init__(self, size, length):\n self.len = length\n self.data = torch.randn(length, size)\n\n def __getitem__(self, index):\n\n # Return data_streams.\n data_streams = DataStreams({\"index\": None})\n data_streams[\"index\"] = self.data[index]\n\n return data_streams\n\n #return self.data[index]\n\n def __len__(self):\n return self.len\n\n def output_data_definitions(self):\n return {\"index\": DataDefinition(1,1,\"str\")}\n\n def collate_fn(self, batch):\n print(\"Collate!\")\n return DataStreams({key: torch.utils.data.dataloader.default_collate([sample[key] for sample in batch]) for key in batch[0]})\n\n\nclass TestModel1(Model):\n # Our model\n def __init__(self, input_size, output_size):\n super(Model, self).__init__()\n self.fc = nn.Linear(input_size, output_size)\n\n def forward(self, datadict):\n input = datadict[\"index\"]\n print(\"Dummy Model: input size {}, device: {}\\n\".format(input.size(), input.device))\n output = self.fc(input)\n print(\"Dummy Model: output size {}\\n\".format(output.size()))\n\n datadict.extend({\"middle\": output})\n #print(\"saved to output : \",type(output))\n #return output\n\n def input_data_definitions(self):\n return {\"index\": DataDefinition(1,1,\"str\")}\n\n def output_data_definitions(self):\n return {\"middle\": DataDefinition(1,1,\"str\")}\n\n\nclass TestModel2(Model):\n # Our model\n def __init__(self, input_size, output_size):\n super(Model, self).__init__()\n self.fc = nn.Linear(input_size, output_size)\n\n def forward(self, datadict):\n input = datadict[\"middle\"]\n\n print(\"Dummy Model: input size {}, device: {}\\n\".format(input.size(), input.device))\n output = self.fc(input)\n print(\"Dummy Model: output size {}\\n\".format(output.size()))\n\n datadict.extend({\"output\": output})\n\n def input_data_definitions(self):\n return {\"middle\": DataDefinition(1,1,\"str\")}\n\n def output_data_definitions(self):\n return {\"output\": DataDefinition(1,1,\"str\")}\n\n\n\nif __name__ == \"__main__\":\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n #device = torch.device('cpu')\n\n use_dataparallel = False\n\n # Parameters and DataLoaders\n input_size = 5\n middle_size = 2\n output_size = 3\n\n batch_size = 10\n data_size = 100000\n\n model1 = TestModel1(input_size, middle_size)\n model2 = TestModel2(middle_size, output_size)\n print(\"Models DONE!!\")\n #time.sleep(2)\n\n dataset = RandomDataset(input_size, data_size)\n rand_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, collate_fn=dataset.collate_fn)\n print(\"Dataloader DONE!!\")\n #time.sleep(2)\n\n\n if torch.cuda.device_count() > 1:\n print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs\n model1 = DataStreamsParallel(model1) \n model2 = DataStreamsParallel(model2) \n use_dataparallel = True\n # Move to desired device.\n model1.to(device)\n model2.to(device)\n print(\"DataParallel DONE!!\")\n #time.sleep(2)\n\n #datadict1 = {}#DataStreams({\"index\":None,\"output\":None})\n for datadict in rand_loader:\n print(\"!!!!! Got object from loader: {}\".format(type(datadict)))\n datadict.to(device)\n\n print(\"datadict before model1: \",datadict)\n if use_dataparallel:\n outputs = model1(datadict)\n # Postprocessing: copy only the outputs of the wrapped model.\n for key in model1.module.output_data_definitions().keys():\n datadict.extend({key: outputs[key]})\n else:\n model1(datadict)\n\n # Let's see what will happen here!\n datadict.to(device)\n\n print(\"datadict before model2: \",datadict)\n if use_dataparallel:\n outputs = model2(datadict)\n # Postprocessing: copy only the outputs of the wrapped model.\n for key in model2.module.output_data_definitions().keys():\n datadict.extend({key: outputs[key]})\n else: \n model2(datadict)\n\n print(\"datadict after model2: \",datadict)\n\n output = datadict[\"output\"]\n loss = output.sum()\n loss.backward()\n\n #print(type(output))\n #output = datadict2[\"output\"]\n #print(\"For: after model: output_size \", output.size(),\"\\n\")\n\n" }, { "path": "ptp/workers/trainer.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Vincent Marois, Tomasz Kornuta\"\n\nfrom os import path,makedirs\nimport yaml\nimport torch\nfrom time import sleep\nfrom datetime import datetime\n\nimport ptp.configuration.config_parsing as config_parse\nimport ptp.utils.logger as logging\n\nfrom ptp.workers.worker import Worker\n\nfrom ptp.application.task_manager import TaskManager\nfrom ptp.application.pipeline_manager import PipelineManager\n\nfrom ptp.utils.statistics_collector import StatisticsCollector\nfrom ptp.utils.statistics_aggregator import StatisticsAggregator\n\n\nclass Trainer(Worker):\n \"\"\"\n Base class for the trainers.\n\n Iterates over epochs on the dataset.\n\n All other types of trainers (e.g. ``OnlineTrainer`` & ``OfflineTrainer``) should subclass it.\n\n \"\"\"\n\n def __init__(self, name, class_type):\n \"\"\"\n Base constructor for all trainers:\n\n - Adds default trainer command line arguments\n\n :param name: Name of the worker\n :type name: str\n\n :param class_type: Class type of the component.\n\n \"\"\" \n # Call base constructor to set up app state, registry and add default arguments.\n super(Trainer, self).__init__(name, class_type)\n\n # Add arguments to the specific parser.\n # These arguments will be shared by all basic trainers.\n self.parser.add_argument(\n '--tensorboard',\n action='store',\n dest='tensorboard', choices=[0, 1, 2],\n type=int,\n help=\"If present, enable logging to TensorBoard. Available log levels:\\n\"\n \"0: Log the collected statistics.\\n\"\n \"1: Add the histograms of the model's biases & weights (Warning: Slow).\\n\"\n \"2: Add the histograms of the model's biases & weights gradients \"\n \"(Warning: Even slower).\")\n\n self.parser.add_argument(\n '--saveall',\n dest='save_intermediate',\n action='store_true',\n help='Setting to true results in saving intermediate models during training (DEFAULT: False)')\n\n self.parser.add_argument(\n '--training',\n dest='training_section_name',\n type=str,\n default=\"training\",\n help='Name of the section defining the training procedure (DEFAULT: training)')\n\n self.parser.add_argument(\n '--validation',\n dest='validation_section_name',\n type=str,\n default=\"validation\",\n help='Name of the section defining the validation procedure (DEFAULT: validation)')\n\n\n def setup_experiment(self):\n \"\"\"\n Sets up experiment of all trainers:\n\n - Calls base class setup_experiment to parse the command line arguments,\n\n - Loads the config file(s)\n\n - Set up the log directory path\n\n - Add a ``FileHandler`` to the logger\n\n - Set random seeds\n\n - Creates the pipeline consisting of many components\n\n - Creates training task manager\n\n - Handles curriculum learning if indicated\n\n - Creates validation task manager\n\n - Set optimizer\n\n - Performs testing of compatibility of both training and validation tasks and created pipeline.\n\n \"\"\"\n # Call base method to parse all command line arguments and add default sections.\n super(Trainer, self).setup_experiment()\n\n # \"Pass\" configuration parameters from the \"default_training\" section to training section indicated by the section_name.\n self.config.add_default_params({ self.app_state.args.training_section_name : self.config['default_training'].to_dict()} )\n self.config.del_default_params('default_training')\n \n # \"Pass\" configuration parameters from the \"default_validation\" section to validation section indicated by the section_name.\n self.config.add_default_params({ self.app_state.args.validation_section_name: self.config['default_validation'].to_dict()} )\n self.config.del_default_params('default_validation')\n\n\n # Check the presence of the CUDA-compatible devices.\n if self.app_state.args.use_gpu and (torch.cuda.device_count() == 0):\n self.logger.error(\"Cannot use GPU as there are no CUDA-compatible devices present in the system!\")\n exit(-1)\n\n # Check if config file was selected.\n if self.app_state.args.config == '':\n print('Please pass configuration file(s) as --c parameter')\n exit(-2)\n\n # Split and make them absolute.\n root_configs = self.app_state.args.config.replace(\" \", \"\").split(',')\n # If there are - expand them to absolute paths.\n abs_root_configs = [path.expanduser(config) for config in root_configs]\n \n # Get the list of configurations which need to be loaded.\n configs_to_load = config_parse.recurrent_config_parse(abs_root_configs, [], self.app_state.absolute_config_path)\n\n # Read the YAML files one by one - but in reverse order -> overwrite the first indicated config(s)\n config_parse.reverse_order_config_load(self.config, configs_to_load)\n\n # -> At this point, the Param Registry contains the configuration loaded (and overwritten) from several files.\n # Log the resulting training configuration.\n conf_str = 'Loaded (initial) configuration:\\n'\n conf_str += '='*80 + '\\n'\n conf_str += yaml.safe_dump(self.config.to_dict(), default_flow_style=False)\n conf_str += '='*80 + '\\n'\n print(conf_str)\n\n # Get training section.\n try:\n tsn = self.app_state.args.training_section_name\n self.config_training = self.config[tsn]\n # We must additionally check if it is None - weird behvaiour when using default value.\n if self.config_training is None:\n raise KeyError()\n except KeyError:\n print(\"Error: Couldn't retrieve the training section '{}' from the loaded configuration\".format(tsn))\n exit(-1)\n\n # Get training task type.\n try:\n training_task_type = self.config_training['task']['type']\n except KeyError:\n print(\"Error: Couldn't retrieve the task 'type' from the training section '{}' in the loaded configuration\".format(tsn))\n exit(-1)\n\n # Get validation section.\n try:\n vsn = self.app_state.args.validation_section_name\n self.config_validation = self.config[vsn]\n if self.config_validation is None:\n raise KeyError()\n except KeyError:\n print(\"Error: Couldn't retrieve the validation section '{}' from the loaded configuration\".format(vsn))\n exit(-1)\n\n # Get validation task type.\n try:\n _ = self.config_validation['task']['type']\n except KeyError:\n print(\"Error: Couldn't retrieve the task 'type' from the validation section '{}' in the loaded configuration\".format(vsn))\n exit(-1)\n\n # Get pipeline section.\n try:\n psn = self.app_state.args.pipeline_section_name\n self.config_pipeline = self.config[psn]\n if self.config_pipeline is None:\n raise KeyError()\n except KeyError:\n print(\"Error: Couldn't retrieve the pipeline section '{}' from the loaded configuration\".format(psn))\n exit(-1)\n\n # Get pipeline name.\n try:\n pipeline_name = self.config_pipeline['name']\n except KeyError:\n # Using name of the first configuration file from command line.\n basename = path.basename(root_configs[0])\n # Take config filename without extension.\n pipeline_name = path.splitext(basename)[0] \n # Set pipeline name, so processor can use it afterwards.\n self.config_pipeline.add_config_params({'name': pipeline_name})\n\n # Prepare the output path for logging\n while True: # Dirty fix: if log_dir already exists, wait for 1 second and try again\n try:\n time_str = '{0:%Y%m%d_%H%M%S}'.format(datetime.now())\n if self.app_state.args.exptag != '':\n time_str = time_str + \"_\" + self.app_state.args.exptag\n self.app_state.log_dir = path.expanduser(self.app_state.args.expdir) + '/' + training_task_type + '/' + pipeline_name + '/' + time_str + '/'\n # Lowercase dir.\n self.app_state.log_dir = self.app_state.log_dir.lower()\n makedirs(self.app_state.log_dir, exist_ok=False)\n except FileExistsError:\n sleep(1)\n else:\n break\n\n # Set log dir.\n self.app_state.log_file = self.app_state.log_dir + 'trainer.log'\n # Initialize logger in app state.\n self.app_state.logger = logging.initialize_logger(\"AppState\")\n # Add handlers for the logfile to worker logger.\n logging.add_file_handler_to_logger(self.logger)\n self.logger.info(\"Logger directory set to: {}\".format(self.app_state.log_dir))\n\n # Set cpu/gpu types.\n self.app_state.set_types()\n\n # Models dir.\n self.checkpoint_dir = self.app_state.log_dir + 'checkpoints/'\n makedirs(self.checkpoint_dir, exist_ok=False)\n\n # Set random seeds in the training section.\n self.set_random_seeds('training', self.config_training)\n\n # Total number of detected errors.\n errors =0\n\n ################# TRAINING PROBLEM ################# \n\n # Build training task manager.\n self.training = TaskManager('training', self.config_training) \n errors += self.training.build()\n \n # parse the curriculum learning section in the loaded configuration.\n if 'curriculum_learning' in self.config_training:\n\n # Initialize curriculum learning - with values from loaded configuration.\n self.training.task.curriculum_learning_initialize(self.config_training['curriculum_learning'])\n\n # If the 'must_finish' key is not present in config then then it will be finished by default\n self.config_training['curriculum_learning'].add_default_params({'must_finish': True})\n\n self.must_finish_curriculum = self.config_training['curriculum_learning']['must_finish']\n self.logger.info(\"Curriculum Learning activated\")\n\n else:\n # If not using curriculum learning then it does not have to be finished.\n self.must_finish_curriculum = False\n self.curric_done = True\n\n ################# VALIDATION PROBLEM ################# \n \n # Build validation task manager.\n self.validation = TaskManager('validation', self.config_validation)\n errors += self.validation.build()\n\n ###################### PIPELINE ######################\n \n # Build the pipeline using the loaded configuration.\n self.pipeline = PipelineManager(pipeline_name, self.config_pipeline)\n errors += self.pipeline.build()\n\n # Check errors.\n if errors > 0:\n self.logger.error('Found {} errors, terminating execution'.format(errors))\n exit(-2)\n\n # Show pipeline.\n summary_str = self.pipeline.summarize_all_components_header()\n summary_str += self.training.task.summarize_io(\"training\")\n summary_str += self.validation.task.summarize_io(\"validation\")\n summary_str += self.pipeline.summarize_all_components()\n self.logger.info(summary_str)\n \n # Handshake definitions.\n self.logger.info(\"Handshaking training pipeline\")\n defs_training = self.training.task.output_data_definitions()\n errors += self.pipeline.handshake(defs_training)\n\n self.logger.info(\"Handshaking validation pipeline\")\n defs_valid = self.validation.task.output_data_definitions()\n errors += self.pipeline.handshake(defs_valid)\n\n # Check errors.\n if errors > 0:\n self.logger.error('Found {} errors, terminating execution'.format(errors))\n exit(-2)\n\n ################## MODEL LOAD/FREEZE #################\n\n # Load the pretrained models params from checkpoint.\n try: \n # Check command line arguments, then check load option in config.\n if self.app_state.args.load_checkpoint != \"\":\n pipeline_name = self.app_state.args.load_checkpoint\n msg = \"command line (--load)\"\n elif \"load\" in self.config_pipeline:\n pipeline_name = self.config_pipeline['load']\n msg = \"'pipeline' section of the configuration file\"\n else:\n pipeline_name = \"\"\n # Try to load the model.\n if pipeline_name != \"\":\n if path.isfile(pipeline_name):\n # Load parameters from checkpoint.\n self.pipeline.load(pipeline_name)\n else:\n raise Exception(\"Couldn't load the checkpoint {} indicated in the {}: file does not exist\".format(pipeline_name, msg))\n # If we succeeded, we do not want to load the models from the file anymore!\n else:\n # Try to load the models parameters - one by one, if set so in the configuration file.\n self.pipeline.load_models()\n\n except KeyError:\n self.logger.error(\"File {} indicated in the {} seems not to be a valid model checkpoint\".format(pipeline_name, msg))\n exit(-5)\n except Exception as e:\n self.logger.error(e)\n # Exit by following the logic: if user wanted to load the model but failed, then continuing the experiment makes no sense.\n exit(-6)\n\n # Finally, freeze the models (that the user wants to freeze).\n self.pipeline.freeze_models()\n\n # Log the model summaries.\n summary_str = self.pipeline.summarize_models_header()\n summary_str += self.pipeline.summarize_models()\n self.logger.info(summary_str)\n\n # Move the models in the pipeline to GPU.\n if self.app_state.args.use_gpu:\n self.pipeline.cuda() \n\n ################# OPTIMIZER ################# \n\n # Set the optimizer.\n optimizer_conf = dict(self.config_training['optimizer'])\n optimizer_type = optimizer_conf['type']\n del optimizer_conf['type']\n\n # Check if there are any models in the pipeline.\n if len(list(filter(lambda p: p.requires_grad, self.pipeline.parameters()))) == 0:\n self.logger.error('Cannot proceed with training, as there are no trainable models in the pipeline (or all models are frozen)')\n exit(-7)\n\n # Instantiate the optimizer and filter the model parameters based on if they require gradients.\n self.optimizer = getattr(torch.optim, optimizer_type)(\n filter(lambda p: p.requires_grad, self.pipeline.parameters()), **optimizer_conf)\n\n log_str = 'Optimizer:\\n' + '='*80 + \"\\n\"\n log_str += \" Type: \" + optimizer_type + \"\\n\"\n log_str += \" Params: {}\".format(optimizer_conf)\n\n self.logger.info(log_str)\n\n def add_statistics(self, stat_col):\n \"\"\"\n Calls base method and adds epoch statistics to ``StatisticsCollector``.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n # Add loss and episode.\n super(Trainer, self).add_statistics(stat_col)\n\n # Add default statistics with formatting.\n stat_col.add_statistics('epoch', '{:02d}')\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds basic aggregators to to ``StatisticsAggregator`` and extends them with: epoch.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n # Add basic aggregators.\n super(Trainer, self).add_aggregators(stat_agg)\n\n # add 'aggregators' for the epoch.\n stat_agg.add_aggregator('epoch', '{:02d}')\n\n\n def initialize_statistics_collection(self):\n \"\"\"\n - Initializes all ``StatisticsCollectors`` and ``StatisticsAggregators`` used by a given worker: \\\n\n - For training statistics (adds the statistics of the model & task),\n - For validation statistics (adds the statistics of the model & task).\n\n - Creates the output files (csv).\n\n \"\"\"\n # TRAINING.\n # Create statistics collector for training.\n self.training_stat_col = StatisticsCollector()\n self.add_statistics(self.training_stat_col)\n self.training.task.add_statistics(self.training_stat_col)\n self.pipeline.add_statistics(self.training_stat_col)\n # Create the csv file to store the training statistics.\n self.training_batch_stats_file = self.training_stat_col.initialize_csv_file(self.app_state.log_dir, 'training_statistics.csv')\n\n # Create statistics aggregator for training.\n self.training_stat_agg = StatisticsAggregator()\n self.add_aggregators(self.training_stat_agg)\n self.training.task.add_aggregators(self.training_stat_agg)\n self.pipeline.add_aggregators(self.training_stat_agg)\n # Create the csv file to store the training statistic aggregations.\n self.training_set_stats_file = self.training_stat_agg.initialize_csv_file(self.app_state.log_dir, 'training_set_agg_statistics.csv')\n\n # VALIDATION.\n # Create statistics collector for validation.\n self.validation_stat_col = StatisticsCollector()\n self.add_statistics(self.validation_stat_col)\n self.validation.task.add_statistics(self.validation_stat_col)\n self.pipeline.add_statistics(self.validation_stat_col)\n # Create the csv file to store the validation statistics.\n self.validation_batch_stats_file = self.validation_stat_col.initialize_csv_file(self.app_state.log_dir, 'validation_statistics.csv')\n\n # Create statistics aggregator for validation.\n self.validation_stat_agg = StatisticsAggregator()\n self.add_aggregators(self.validation_stat_agg)\n self.validation.task.add_aggregators(self.validation_stat_agg)\n self.pipeline.add_aggregators(self.validation_stat_agg)\n # Create the csv file to store the validation statistic aggregations.\n self.validation_set_stats_file = self.validation_stat_agg.initialize_csv_file(self.app_state.log_dir, 'validation_set_agg_statistics.csv')\n\n\n def finalize_statistics_collection(self):\n \"\"\"\n Finalizes the statistics collection by closing the csv files.\n\n \"\"\"\n # Close all files.\n self.training_batch_stats_file.close()\n self.training_set_stats_file.close()\n self.validation_batch_stats_file.close()\n self.validation_set_stats_file.close()\n\n\n def initialize_tensorboard(self):\n \"\"\"\n Initializes the TensorBoard writers, and log directories.\n\n \"\"\"\n # Create TensorBoard outputs - if TensorBoard is supposed to be used.\n if self.app_state.args.tensorboard is not None:\n from tensorboardX import SummaryWriter\n self.training_batch_writer = SummaryWriter(self.app_state.log_dir + '/training')\n self.training_stat_col.initialize_tensorboard(self.training_batch_writer)\n\n self.training_set_writer = SummaryWriter(self.app_state.log_dir + '/training_set_agg')\n self.training_stat_agg.initialize_tensorboard(self.training_set_writer)\n \n self.validation_batch_writer = SummaryWriter(self.app_state.log_dir + '/validation')\n self.validation_stat_col.initialize_tensorboard(self.validation_batch_writer)\n\n self.validation_set_writer = SummaryWriter(self.app_state.log_dir + '/validation_set_agg')\n self.validation_stat_agg.initialize_tensorboard(self.validation_set_writer)\n else:\n self.training_batch_writer = None\n self.training_set_writer = None\n self.validation_batch_writer = None\n self.validation_set_writer = None\n\n def finalize_tensorboard(self):\n \"\"\" \n Finalizes the operation of TensorBoard writers by closing them.\n \"\"\"\n # Close the TensorBoard writers.\n if self.training_batch_writer is not None:\n self.training_batch_writer.close()\n if self.training_set_writer is not None:\n self.training_set_writer.close()\n if self.validation_batch_writer is not None:\n self.validation_batch_writer.close()\n if self.validation_set_writer is not None:\n self.validation_set_writer.close()\n\n def validate_on_batch(self, valid_batch):\n \"\"\"\n Performs a validation of the model using the provided batch.\n\n Additionally logs results (to files, TensorBoard) and handles visualization.\n\n :param valid_batch: data batch generated by the task and used as input to the model.\n :type valid_batch: ``DataStreams``\n\n :return: Validation loss.\n\n \"\"\"\n # Turn on evaluation mode.\n self.pipeline.eval()\n # Empty the statistics collector.\n self.validation_stat_col.empty()\n\n # Compute the validation loss using the provided data batch.\n with torch.no_grad():\n # Forward pass.\n self.pipeline.forward(valid_batch)\n # Collect the statistics.\n self.collect_all_statistics(self.validation, self.pipeline, valid_batch, self.validation_stat_col)\n\n # Export collected statistics.\n self.export_all_statistics(self.validation_stat_col, '[Partial Validation]')\n\n def validate_on_set(self):\n \"\"\"\n Performs a validation of the model on the whole validation set, using the validation ``DataLoader``.\n\n Iterates over the entire validation set (through the `DataLoader``), aggregates the collected statistics \\\n and logs that to the console, csv and TensorBoard (if set).\n\n \"\"\"\n # Get number of samples.\n num_samples = len(self.validation)\n \n self.logger.info('Validating over the entire validation set ({} samples in {} episodes)'.format(\n num_samples, len(self.validation.dataloader)))\n\n # Turn on evaluation mode.\n self.pipeline.eval()\n\n # Reset the statistics.\n self.validation_stat_col.empty()\n\n # Remember global episode number.\n old_episode = self.app_state.episode\n\n with torch.no_grad():\n for ep, valid_batch in enumerate(self.validation.dataloader):\n\n self.app_state.episode = ep\n # Forward pass.\n self.pipeline.forward(valid_batch)\n # Collect the statistics.\n self.collect_all_statistics(self.validation, self.pipeline, valid_batch,\n self.validation_stat_col)\n\n # Revert to global episode number.\n self.app_state.episode = old_episode\n\n # Aggregate statistics for the whole set.\n self.aggregate_all_statistics(self.validation, self.pipeline,\n self.validation_stat_col, self.validation_stat_agg)\n\n # Export aggregated statistics.\n self.export_all_statistics(self.validation_stat_agg, '[Full Validation]')\n\n\nif __name__ == '__main__':\n print(\"The trainer.py file contains only an abstract base class. Please try to use the \\\nonline_trainer (mip-online-trainer) or offline_trainer (mip-offline-trainer) instead.\")\n" }, { "path": "ptp/workers/worker.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Vincent Marois, Tomasz Kornuta, Ryan L. McAvoy\"\n\nimport torch\nimport argparse\nimport numpy as np\nfrom random import randrange\nfrom abc import abstractmethod\n\nimport ptp.utils.logger as logging\nfrom ptp.utils.app_state import AppState\n\nfrom ptp.configuration.config_interface import ConfigInterface\nfrom ptp.configuration.config_parsing import load_class_default_config_file\n\n\nclass Worker(object):\n \"\"\"\n Base abstract class for the workers.\n All base workers should subclass it and override the relevant methods.\n \"\"\"\n\n def __init__(self, name, class_type, add_default_parser_args = True):\n \"\"\"\n Base constructor for all workers:\n\n - Initializes the AppState singleton\n - Initializes the Configuration Registry\n - Loads default parameters\n - Creates parser and adds default worker command line arguments\n\n :param name: Name of the worker.\n :type name: str\n\n :param class_type: Class type of the component.\n\n :param add_default_parser_args: If set, adds default parser arguments (DEFAULT: True).\n :type add_default_parser_args: bool\n\n \"\"\"\n # Call base constructor.\n super(Worker, self).__init__()\n\n # Set worker name.\n self.name = name\n\n # Initialize the application state singleton.\n self.app_state = AppState()\n\n # Initialize parameter interface/registry.\n self.config = ConfigInterface()\n\n # Load default configuration.\n if class_type is not None:\n self.config.add_default_params(load_class_default_config_file(class_type))\n\n\n # Create parser with a list of runtime arguments.\n self.parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)\n\n # Add arguments to the specific parser.\n if add_default_parser_args:\n # These arguments will be shared by all basic workers.\n self.parser.add_argument(\n '--config',\n dest='config',\n type=str,\n default='',\n help='Name of the configuration file(s) to be loaded. '\n 'If specifying more than one file, they must be separated with coma \",\".')\n\n self.parser.add_argument(\n '--disable',\n type=str,\n default='',\n dest='disable',\n help='Comma-separated list of components to be disabled (DEFAULT: empty)')\n\n self.parser.add_argument(\n '--load',\n type=str,\n default='',\n dest='load_checkpoint',\n help='Path and name of the checkpoint file containing the saved parameters'\n ' of the pipeline models to load (should end with a .pt extension)')\n\n self.parser.add_argument(\n '--gpu',\n dest='use_gpu',\n action='store_true',\n help='The current worker will move the computations on GPU devices, if available '\n 'in the system. (Default: False)')\n\n self.parser.add_argument(\n '--expdir',\n dest='expdir',\n type=str,\n default=\"~/experiments\",\n help='Path to the directory where the experiment(s) folders are/will be stored.'\n ' (DEFAULT: ~/experiments)')\n\n self.parser.add_argument(\n '--exptag',\n dest='exptag',\n type=str,\n default='',\n help=\"Additional tag that will be added to the output folder name (DEFAULT: '').\")\n\n self.parser.add_argument(\n '--logger',\n action='store',\n dest='log_level',\n type=str,\n default='INFO',\n choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'],\n help=\"Log level. (DEFAULT: INFO)\")\n\n self.parser.add_argument(\n '--interval',\n dest='logging_interval',\n default=100,\n type=int,\n help='Statistics logging interval. Will impact logging to the logger and '\n 'exporting to TensorBoard. Writing to the csv file is not impacted '\n '(exports at every step). (DEFAULT: 100, i.e. logs every 100 episodes).')\n\n self.parser.add_argument(\n '--agree',\n dest='confirm',\n action='store_true',\n help='Request user confirmation just after loading the settings, '\n 'before starting the experiment. (DEFAULT: False)')\n\n self.parser.add_argument(\n '--pipeline',\n dest='pipeline_section_name',\n type=str,\n default=\"pipeline\",\n help='Name of the section defining the pipeline (DEFAULT: pipeline)')\n\n\n def setup_experiment(self):\n \"\"\"\n Setups a specific experiment.\n\n Base method:\n\n - Parses command line arguments.\n\n - Initializes logger with worker name.\n\n - Sets the 3 default config sections (training / validation / test) and sets their dataloaders params.\n\n .. note::\n\n Child classes should override this method, but still call its parent to draw the basic functionality \\\n implemented here.\n\n\n \"\"\"\n # Parse arguments.\n self.app_state.args, self.unparsed = self.parser.parse_known_args()\n\n # Initialize logger using the configuration.\n # For now do not add file handler, as path to logfile is not known yet.\n self.logger = logging.initialize_logger(self.name, False)\n\n\n def add_statistics(self, stat_col):\n \"\"\"\n Adds most elementary shared statistics to ``StatisticsCollector``: episode.\n\n :param stat_col: ``StatisticsCollector``.\n\n \"\"\"\n # Add default statistics with formatting.\n stat_col.add_statistics('episode', '{:06d}')\n\n\n def add_aggregators(self, stat_agg):\n \"\"\"\n Adds basic statistical aggregators to ``StatisticsAggregator``: episode \\\n episodes_aggregated.\n\n :param stat_agg: ``StatisticsAggregator``.\n\n \"\"\"\n # add 'aggregators' for the episode.\n #stat_agg.add_aggregator('epoch', '{:02d}')\n stat_agg.add_aggregator('episode', '{:06d}')\n # Number of aggregated episodes.\n stat_agg.add_aggregator('episodes_aggregated', '{:06d}')\n\n\n @abstractmethod\n def run_experiment(self):\n \"\"\"\n Main function of the worker which executes a specific experiment.\n\n .. note::\n\n Abstract. Should be implemented in the subclasses.\n \"\"\"\n\n\n def collect_all_statistics(self, task_mgr, pipeline_mgr, data_streams, stat_col):\n \"\"\"\n Function that collects statistics\n\n :param pipeline: Pipeline containing both task and list of components.\n :type pipeline: ``configuration.pipeline.Pipeline``\n\n :param task_mgr: Task manager.\n\n :param data_streams: contains the batch of samples to pass through the pipeline.\n :type data_streams: ``DataStreams``\n\n :param stat_col: statistics collector used for logging accuracy etc.\n :type stat_col: ``StatisticsCollector``\n\n \"\"\"\n # Collect \"local\" statistics.\n stat_col['episode'] = self.app_state.episode\n if ('epoch' in stat_col) and (self.app_state.epoch is not None):\n stat_col['epoch'] = self.app_state.epoch\n\n # Collect rest of statistics.\n task_mgr.task.collect_statistics(stat_col, data_streams)\n pipeline_mgr.collect_statistics(stat_col, data_streams)\n\n \n\n def aggregate_all_statistics(self, task_mgr, pipeline_mgr, stat_col, stat_agg):\n \"\"\"\n Aggregates the collected statistics. Exports the aggregations to logger, csv and TB. \\\n Empties statistics collector for the next episode.\n\n :param pipeline: Pipeline containing both task and list of components.\n :type pipeline: ``configuration.pipeline.Pipeline``\n\n :param task_mgr: Task manager.\n\n :param stat_col: ``StatisticsCollector`` object.\n\n :param stat_agg: ``StatisticsAggregator`` object.\n \"\"\" \n # Aggregate \"local\" statistics.\n if ('epoch' in stat_col) and ('epoch' in stat_agg) and (self.app_state.epoch is not None):\n stat_agg.aggregators['epoch'] = self.app_state.epoch\n stat_agg.aggregators['episode'] = self.app_state.episode\n stat_agg.aggregators['episodes_aggregated'] = len(stat_col['episode'])\n # Aggregate rest of statistics.\n task_mgr.task.aggregate_statistics(stat_col, stat_agg)\n pipeline_mgr.aggregate_statistics(stat_col, stat_agg)\n \n\n def export_all_statistics(self, stat_obj, tag='', export_to_log = True):\n \"\"\"\n Export the statistics/aggregations to logger, csv and TB.\n\n :param stat_obj: ``StatisticsCollector`` or ``StatisticsAggregato`` object.\n\n :param tag: Additional tag that will be added to string exported to logger, optional (DEFAULT = '').\n :type tag: str\n\n :param export_to_log: If True, exports statistics to logger (DEFAULT: True)\n :type export_to_log: bool\n\n \"\"\" \n # Log to logger\n if export_to_log:\n self.logger.info(stat_obj.export_to_string(tag))\n\n # Export to csv\n stat_obj.export_to_csv()\n\n # Export to TensorBoard.\n stat_obj.export_to_tensorboard()\n\n\n def set_random_seeds(self, section_name, config):\n \"\"\"\n Set ``torch`` & ``NumPy`` random seeds from the ``ParamRegistry``: \\\n If one was indicated, use it, or set a random one.\n\n :param section_name: Name of the section (for logging purposes only).\n :type section_name: str\n\n :param config: Section in config registry that will be changed \\\n (\"training\" or \"testing\" only will be taken into account.)\n\n \"\"\"\n # Set the random seeds: either from the loaded configuration or a default randomly selected one.\n if config[\"seed_numpy\"] == -1:\n seed = randrange(0, 2 ** 32)\n # Overwrite the config param!\n config.add_config_params({\"seed_numpy\": seed})\n\n self.logger.info(\"Setting numpy random seed in {} to: {}\".format(section_name, config[\"seed_numpy\"]))\n np.random.seed(config[\"seed_numpy\"])\n\n if config[\"seed_torch\"] == -1:\n seed = randrange(0, 2 ** 32)\n # Overwrite the config param!\n config.add_config_params({\"seed_torch\": seed})\n\n self.logger.info(\"Setting torch random seed in {} to: {}\".format(section_name, config[\"seed_torch\"]))\n torch.manual_seed(config[\"seed_torch\"])\n torch.cuda.manual_seed_all(config[\"seed_torch\"])\n" }, { "path": "setup.py", "content": "# Copyright (C) IBM Corporation 2019-2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# Always prefer setuptools over distutils\nfrom setuptools import setup, find_packages\nfrom os import path,makedirs\n# io.open is needed for projects that support Python 2.7\n# It ensures open() defaults to text mode with universal newlines,\n# and accepts an argument to specify the text encoding\n# Python 3 only projects can skip this import\n# from io import open\n\nhere = path.abspath(path.dirname(__file__))\n\n# Get path to configs.\nconfigs_path = path.join(here,\"configs/\")\n# Export path to config file in ~/.ptp/ folder.\nptp_path = path.expanduser(\"~/.ptp/\")\n# Make dir.\nmakedirs(path.dirname(ptp_path), exist_ok=True)\n# Write path to configs.\nwith open(path.join(ptp_path, \"config.txt\"),\"w\") as file:\n file.write(configs_path)\n\n# Get the long description from the README file\nwith open(path.join(here, 'README.md'), encoding='utf-8') as f:\n long_description = f.read()\n\n# Arguments marked as \"Required\" below must be included for upload to PyPI.\n# Fields marked as \"Optional\" may be commented out.\n\nsetup(\n # This is the name of your project. The first time you publish this\n # package, this name will be registered for you. It will determine how\n # users can install this project, e.g.:\n #\n # $ pip install sampleproject\n #\n # And where it will live on PyPI: https://pypi.org/project/sampleproject/\n #\n # There are some restrictions on what makes a valid project name\n # specification here:\n # https://packaging.python.org/specifications/core-metadata/#name\n name='ptp', # Required\n\n # Versions should comply with PEP 440:\n # https://www.python.org/dev/peps/pep-0440/\n #\n # For a discussion on single-sourcing the version across setup.py and the\n # project code, see\n # https://packaging.python.org/en/latest/single_source_version.html\n version='0.1', # Required\n\n # This is a one-line description or tagline of what your project does. This\n # corresponds to the \"Summary\" metadata field:\n # https://packaging.python.org/specifications/core-metadata/#summary\n description='PyTorchPipe: framework for building multi-modal PyTorch pipelines',\n\n # This is an optional longer description of your project that represents\n # the body of text which users will see when they visit PyPI.\n #\n # Often, this is the same as your README, so you can just read it in from\n # that file directly (as we have already done above)\n #\n # This field corresponds to the \"Description\" metadata field:\n # https://packaging.python.org/specifications/core-metadata/#description-optional\n long_description=long_description, # Optional\n\n # Denotes that our long_description is in Markdown; valid values are\n # text/plain, text/x-rst, and text/markdown\n #\n # Optional if long_description is written in reStructuredText (rst) but\n # required for plain-text or Markdown; if unspecified, \"applications should\n # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and\n # fall back to text/plain if it is not valid rst\" (see link below)\n #\n # This field corresponds to the \"Description-Content-Type\" metadata field:\n # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional\n long_description_content_type='text/markdown', # Optional (see note above)\n\n # This should be a valid link to your project's main homepage.\n #\n # This field corresponds to the \"Home-Page\" metadata field:\n # https://packaging.python.org/specifications/core-metadata/#home-page-optional\n url='https://github.com/IBM/pytorchpipe/', # Optional\n license='Apache 2.0',\n\n # This should be your name or the name of the organization which owns the\n # project.\n author='Tomasz Kornuta', # Optional\n\n # This should be a valid email address corresponding to the author listed\n # above.\n author_email='tkornuta@us.ibm.com', # Optional\n\n # Classifiers help users find your project by categorizing it.\n #\n # For a list of valid classifiers, see https://pypi.org/classifiers/\n # This information is only used for searching & browsing projects on PyPI, not for installing projects\n # Checkout numpy: https://pypi.org/project/numpy/\n\n classifiers=[ # Optional\n # How mature is this project? Common values are\n # 3 - Alpha\n # 4 - Beta\n # 5 - Production/Stable\n 'Development Status :: 0 - Pre Pre Alfa',\n\n # Indicate who your project is intended for\n 'Intended Audience :: Science/Research',\n 'Intended Audience :: Developers',\n\n # Pick your license as you wish\n 'License :: OSI Approved :: Apache Software License',\n\n # Specify the Python versions you support here. In particular, ensure\n # that you indicate whether you support Python 2, Python 3 or both.\n 'Programming Language :: Python :: 3.6',\n\n 'Operating System :: Linux',\n 'Topic :: Scientific/Engineering :: Artificial Intelligence'\n ],\n\n # This field adds keywords for your project which will appear on the\n # project page. What does your project relate to?\n #\n # Note that this is a string of words separated by whitespace, not a list.\n keywords='machine learning neural nets pytorch pipeline component task model', # Optional\n\n # You can just specify package directories manually here if your project is\n # simple. Or you can use find_packages().\n #\n # Alternatively, if you just want to distribute a single Python file, use\n # the `py_modules` argument instead as follows, which will expect a file\n # called `my_module.py` to exist:\n #\n # py_modules=[\"my_module\"],\n #\n packages=find_packages(exclude=['docs', 'configs', 'build', 'experiments', 'scripts']), # Required\n\n # This field lists other packages that your project depends on to run.\n # Any package you put here will be installed by pip when your project is\n # installed, so they must be valid existing projects.\n #\n python_requires='~=3.6',\n # For an analysis of \"install_requires\" vs pip's requirements files see:\n # https://packaging.python.org/en/latest/requirements.html\n # Should not pin down version\n # It is not considered best practice to use install_requires to pin\n # dependencies to specific versions, or to specify sub-dependencies\n # (i.e. dependencies of your dependencies). This is overly-restrictive,\n # and prevents the user from gaining the benefit of dependency upgrades.\n install_requires=[\n 'tqdm',\n 'nltk',\n 'pandas',\n 'pillow',\n 'torchtext==0.3.1',\n 'torchvision==0.2.1',\n 'torch>=1.0.1',\n 'PyYAML',\n 'matplotlib',\n 'requests'\n ],\n\n # List additional groups of dependencies here (e.g. development\n # dependencies). Users will be able to install these using the \"extras\"\n # syntax, for example:\n #\n # $ pip install sampleproject[dev]\n #\n # Similar to `install_requires` above, these must be valid existing\n # projects.\n extras_require={ # Optional\n # 'dev': ['tensorflow', 'ipdb', 'tensorboard', 'visdom', 'tensorboardX'],\n # 'test': ['coverage'],\n },\n\n # If there are data files included in your packages that need to be\n # installed, specify them here.\n #\n # If using Python 2.6 or earlier, then these have to be included in\n # MANIFEST.in as well.\n package_data={},\n\n include_package_data=True,\n\n # Although 'package_data' is the preferred approach, in some case you may\n # need to place data files outside of your packages. See:\n # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files\n #\n # In this case, 'data_file' will be installed into '/my_data'\n # data_files=[('my_data', ['data/data_file'])], # Optional\n\n # To provide executable scripts, use entry points in preference to the\n # \"scripts\" keyword. Entry points provide cross-platform support and allow\n # `pip` to create the appropriate form of executable for the target\n # platform.\n #\n # For example, the following would provide a command called `sample` which\n # executes the function `main` from this package when invoked:\n entry_points={ # Optional\n 'console_scripts': [\n 'ptp-online-trainer=ptp.workers.online_trainer:main',\n 'ptp-offline-trainer=ptp.workers.offline_trainer:main',\n 'ptp-processor=ptp.workers.processor:main',\n ]\n },\n\n # List additional URLs that are relevant to your project as a dict.\n #\n # This field corresponds to the \"Project-URL\" metadata fields:\n # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use\n #\n # Examples listed include a pattern for specifying where the package tracks\n # issues, where the source is hosted, where to say thanks to the package\n # maintainers, and where to support the project financially. The key is\n # what's used to render the link text on PyPI.\n project_urls={ # Optional\n 'Source': 'https://github.com/tkornut/pytorchpipe/',\n },\n)\n" }, { "path": "tests/__init__.py", "content": "from .application.pipeline_tests import TestPipeline\nfrom .application.sampler_factory_tests import TestSamplerFactory\nfrom .application.samplers_tests import TestkFoldRandomSampler, TestkFoldWeightedRandomSampler\n\nfrom .components.component_tests import TestComponent\nfrom .components.tasks.clevr_tests import TestCLEVR\nfrom .components.tasks.gqa_tests import TestGQA\nfrom .components.tasks.task_tests import TestTask\n\nfrom .configuration.config_interface_tests import TestConfigInterface\nfrom .configuration.config_registry_tests import TestConfigRegistry\nfrom .configuration.handshaking_tests import TestHandshaking\n\nfrom .data_types.data_streams_tests import TestDataStreams\nfrom .data_types.data_definition_tests import TestDataDefinition\n\nfrom .utils.app_state_tests import TestAppState\nfrom .utils.statistics_tests import TestStatistics\n\n__all__ = [\n # Application\n 'TestPipeline',\n 'TestSamplerFactory',\n 'TestkFoldRandomSampler',\n 'TestkFoldWeightedRandomSampler',\n # Components\n 'TestComponent',\n 'TestGQA',\n 'TestTask',\n # Configuration\n 'TestConfigRegistry',\n 'TestConfigInterface',\n 'TestHandshaking',\n # DataTypes\n 'TestDataStreams',\n 'TestDataDefinition',\n # Utils\n 'TestAppState',\n 'TestStatistics',\n ]\n" }, { "path": "tests/application/pipeline_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nimport os\n\nfrom ptp.utils.app_state import AppState\nfrom ptp.configuration.config_interface import ConfigInterface\nfrom ptp.configuration.config_registry import ConfigRegistry\nfrom ptp.application.pipeline_manager import PipelineManager\n\nclass TestPipeline(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestPipeline, self).__init__(*args, **kwargs)\n # Set required globals.\n app_state = AppState()\n app_state.__setitem__(\"bow_size\", 10, override=True)\n \n def test_create_component_full_type(self):\n \"\"\" Tests whether component can be created when using full module name with 'path'. \"\"\"\n # Instantiate.\n ConfigRegistry()._clear_registry()\n config = ConfigInterface()\n config.add_default_params({\n 'bow_encoder' : \n {\n 'type': 'ptp.components.language.bow_encoder.BOWEncoder',\n 'priority': 1.1\n }\n })\n # Build object.\n pipe = PipelineManager('testpm', config)\n pipe.build(False)\n\n # Assert type.\n self.assertEqual(type(pipe[0]).__name__, \"BOWEncoder\")\n\n\n def test_create_component_type(self):\n \"\"\" Tests whether component can be created when using only module name. \"\"\"\n # Instantiate.\n ConfigRegistry()._clear_registry()\n config = ConfigInterface()\n config.add_default_params({\n 'bow_encoder' : \n {\n 'type': 'BOWEncoder',\n 'priority': 1.2\n }\n })\n # Build object.\n pipe = PipelineManager('testpm', config)\n pipe.build(False)\n\n # Assert type.\n self.assertEqual(type(pipe[0]).__name__, \"BOWEncoder\")\n\n\n def test_disable_component(self):\n \"\"\" Tests whether skipping (disable) works properly. \"\"\"\n # Set param registry.\n ConfigRegistry()._clear_registry()\n config = ConfigInterface()\n config.add_default_params({\n 'disable': 'bow_encoder',\n 'bow_encoder' : \n {\n 'type': 'BOWEncoder',\n 'priority': 1\n }\n })\n # Build object.\n pipe = PipelineManager('testpm', config)\n pipe.build(False)\n\n # Assert no components were created.\n self.assertEqual(len(pipe), 0)\n\n\n def test_priorities(self):\n \"\"\" Tests component priorities. \"\"\"\n # Instantiate.\n ConfigRegistry()._clear_registry()\n config = ConfigInterface()\n config.add_default_params({\n 'bow_encoder2' : \n {\n 'type': 'BOWEncoder',\n 'priority': 2.1\n },\n 'bow_encoder1' : \n {\n 'type': 'BOWEncoder',\n 'priority': 0.1\n }\n })\n pipe = PipelineManager('testpm', config)\n pipe.build(False)\n\n # Assert the right order of components.\n self.assertEqual(len(pipe), 2)\n self.assertEqual(pipe[0].name, 'bow_encoder1')\n self.assertEqual(pipe[1].name, 'bow_encoder2')\n\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/application/sampler_factory_tests.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nimport yaml\nimport numpy as np\n\nfrom ptp.configuration.config_interface import ConfigInterface\nfrom ptp.application.sampler_factory import SamplerFactory\n\n# Task.\nclass TestTaskMockup(object):\n def __len__(self):\n return 50\n\nclass TestSamplerFactory(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestSamplerFactory, self).__init__(*args, **kwargs)\n\n def test_create_subset_random_sampler_range(self):\n \"\"\" Tests whther SubsetRandomSampler accepts 'indices' with the option 1: range. \"\"\"\n\n indices = range(20)\n config = ConfigInterface()\n config.add_default_params({'type': 'SubsetRandomSampler',\n 'indices': indices})\n # Create the sampler.\n sampler = SamplerFactory.build(TestTaskMockup(), config, \"training\")\n\n # Check number of samples.\n self.assertEqual(len(sampler), 20)\n\n def test_create_subset_random_sampler_range_str(self):\n \"\"\" Tests whther SubsetRandomSampler accepts 'indices' with the option 2: range as str. \"\"\"\n\n range_str = '0, 20'\n config = ConfigInterface()\n config.add_default_params({'type': 'SubsetRandomSampler',\n 'indices': range_str})\n # Create the sampler.\n sampler = SamplerFactory.build(TestTaskMockup(), config, \"training\")\n\n # Check number of samples.\n self.assertEqual(len(sampler), 20)\n \n\n def test_create_subset_random_sampler_list_of_indices(self):\n \"\"\" Tests whther SubsetRandomSampler accepts 'indices' with the option 3: list of indices. \"\"\"\n\n yaml_list = yaml.safe_load('[0, 2, 5, 10]')\n config = ConfigInterface()\n config.add_default_params({'type': 'SubsetRandomSampler',\n 'indices': yaml_list})\n # Create the sampler.\n sampler = SamplerFactory.build(TestTaskMockup(), config, \"training\")\n\n # Check number of samples.\n self.assertEqual(len(sampler), 4)\n\n\n def test_create_subset_random_sampler_file(self):\n \"\"\" Tests whther SubsetRandomSampler accepts 'indices' with the option 4: name of the file containing indices. \"\"\"\n\n filename = \"/tmp/tmp_indices.txt\"\n # Store indices to file.\n indices = np.asarray([1,2,3,4,5],dtype=int)\n # Write array to file, separate elements with commas.\n indices.tofile(filename, sep=',', format=\"%s\")\n\n config = ConfigInterface()\n config.add_default_params({'type': 'SubsetRandomSampler',\n 'indices': filename})\n # Create the sampler.\n sampler = SamplerFactory.build(TestTaskMockup(), config, \"training\")\n\n # Check number of samples.\n self.assertEqual(len(sampler), 5)\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/application/samplers_tests.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nimport yaml\nimport numpy as np\n\nfrom ptp.configuration.config_interface import ConfigInterface\nfrom ptp.utils.samplers import kFoldRandomSampler, kFoldWeightedRandomSampler\n\nclass TestkFoldRandomSampler(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestkFoldRandomSampler, self).__init__(*args, **kwargs)\n\n def test_kfold_random_sampler_current_fold(self):\n \"\"\" Tests the k-fold sampler current_fold mode. \"\"\"\n\n # Create the sampler.\n sampler = kFoldRandomSampler(20, 3, all_but_current_fold=False)\n\n # Test zero-th fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 7)\n # Check presence of all indices.\n for ix in range(0,7):\n self.assertIn(ix, indices)\n\n # Test first fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 7)\n # Check presence of all indices.\n for ix in range(7,14):\n self.assertIn(ix, indices)\n\n # Test second fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 6)\n # Check presence of all indices.\n for ix in range(14,20):\n self.assertIn(ix, indices)\n\n # Test third (i.e. zero-th for the second time) fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 7)\n # Check presence of all indices.\n for ix in range(0,7):\n self.assertIn(ix, indices)\n\n\n def test_kfold_random_sampler_current_fold_10epochs(self):\n \"\"\" Tests the k-fold sampler current_fold mode. \"\"\"\n\n # Create the sampler.\n sampler = kFoldRandomSampler(20, 3, 10, all_but_current_fold=False)\n\n # First 10 epochs - the same indices from 0-7 range.\n for _ in range(10):\n # Test zero-th fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 7)\n # Check presence of all indices.\n for ix in range(0,7):\n self.assertIn(ix, indices)\n\n # Next 10 epochs - the same indices from 7-14 range.\n for _ in range(10):\n # Test zero-th fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 7)\n # Check presence of all indices.\n for ix in range(7,14):\n self.assertIn(ix, indices)\n\n\n def test_kfold_random_sampler_all_but_current_fold(self):\n \"\"\" Tests the k-fold sampler all_but_current_fold mode. \"\"\"\n\n # Create the sampler.\n sampler = kFoldRandomSampler(20, 3, all_but_current_fold=True)\n\n # Test zero-th fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 13)\n self.assertEqual(len(sampler), 13)\n \n # Check presence of all indices.\n for ix in range(7,20):\n self.assertIn(ix, indices)\n\n # Test first fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 13)\n # Check presence of all indices.\n for ix in [*range(0,7), *range(14,20)]:\n self.assertIn(ix, indices)\n\n # Test second fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 14)\n # Check presence of all indices.\n for ix in range(0,14):\n self.assertIn(ix, indices)\n\n # Test third (i.e. zero-th for the second time) fold.\n indices = list(iter(sampler))\n # Check number of samples.\n self.assertEqual(len(indices), 13)\n # Check presence of all indices.\n for ix in range(7,20):\n self.assertIn(ix, indices)\n\n\nclass TestkFoldWeightedRandomSampler(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestkFoldWeightedRandomSampler, self).__init__(*args, **kwargs)\n\n def test_kfold_weighed_random_sampler_current_fold(self):\n \"\"\" Tests the k-fold sampler current_fold mode. \"\"\"\n # Non-uniform weights.\n weights = [0.5]*2 + [0] * 2 + [0.5] + [0] * 2 +[0.5]\n\n # Create the sampler.\n sampler = kFoldWeightedRandomSampler(weights, 8, 2, all_but_current_fold=False)\n\n # Test zero-th fold.\n indices = list(iter(sampler))\n #print(indices)\n\n # Check that the rights indices are there.\n for ix in indices:\n self.assertIn(ix, [0,1])\n\n # Test first fold.\n indices = list(iter(sampler))\n #print(indices)\n\n # Check that the rights indices are there.\n for ix in indices:\n self.assertIn(ix, [4,7])\n\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/components/component_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.components.component import Component\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.configuration.config_interface import ConfigInterface\n\nclass MockupTask (Task):\n \"\"\"\n Mockup task class.\n \"\"\"\n def __init__(self, name, config):\n Task.__init__(self, name, None, config)\n\n def output_data_definitions(self):\n return {\n \"inputs\": DataDefinition([-1, 1], [list, int], \"inputs\"),\n }\n\n\nclass MockupComponent (Component):\n \"\"\"\n Mockup component class.\n \"\"\"\n def __init__(self, name, config):\n Component.__init__(self, name, None, config)\n\n\nclass TestComponent(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestComponent, self).__init__(*args, **kwargs)\n\n # Overwrite abc abstract methods.\n MockupComponent.__abstractmethods__=set()\n MockupTask.__abstractmethods__=set()\n\n # Create mocked-up component.\n config = ConfigInterface()\n self.task = MockupTask(\"test_task\", config)\n self.component = MockupComponent(\"test_component\", config)\n\n def test_create_data_streams_key_present(self):\n \"\"\" Tests whether the created data dict contains required keys. \"\"\"\n data_streams = self.task.create_data_streams(1)\n # Check presence of index.\n self.assertEqual(data_streams['indices'], 1) # Even if we didn't explicitly indicated that in definitions!\n self.assertEqual(data_streams['inputs'], None)\n # And targets is not present (yet)...\n with self.assertRaises(KeyError):\n data_streams['targets']\n\n def test_extend_data_streams_key_present(self):\n \"\"\" Tests whether the created data dict contains required keys. \"\"\"\n data_streams = self.task.create_data_streams(1)\n # Extend data_streams.\n data_streams.publish({\"targets\": 3})\n\n # Check presence of all \"streams\".\n self.assertEqual(data_streams['indices'], 1) # Even if we didn't explicitly indicated that in definitions!\n self.assertEqual(data_streams['inputs'], None)\n self.assertEqual(data_streams['targets'], 3)\n\n def test_global_set_get(self):\n \"\"\" Tests setting and getting global value. \"\"\"\n # Set global value.\n self.component.globals[\"embeddings_size\"] = 10\n # Get global value.\n self.assertEqual(self.component.globals[\"embeddings_size\"], 10)\n\n def test_global_overwrite(self):\n \"\"\" Tests global value overwrite \"\"\"\n # Set global value.\n self.component.globals[\"value\"] = \"ala\"\n # Overwrite with the same value - ok.\n self.component.globals[\"value\"] = \"ala\"\n # Overwrite with the same value - error.\n with self.assertRaises(KeyError):\n self.component.globals[\"value\"] = \"ola\"\n\n" }, { "path": "tests/components/tasks/clevr_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nfrom unittest.mock import MagicMock, patch\nfrom os import path\n\nfrom ptp.components.mixins.io import check_file_existence\nfrom ptp.components.tasks.image_text_to_class.clevr import CLEVR\nfrom ptp.configuration.config_interface import ConfigInterface\n\n\nclass TestCLEVR(unittest.TestCase):\n\n def test_training_set(self):\n \"\"\"\n Tests the training split.\n\n ..note:\n Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_train_questions.json' is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface(\"CLEVR\")\n config.add_config_params({\"clevr_training\": {\"split\": \"training\", \"globals\": {\"image_height\": \"clevr_image_height\", \"image_width\": \"clevr_image_width\"}}})\n\n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json'):\n\n # Create object.\n clevr = CLEVR(\"clevr_training\", config[\"clevr_training\"])\n \n # Check dataset size.\n self.assertEqual(len(clevr), 699989)\n\n # Get sample.\n sample = clevr[0]\n\n else: \n dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, \n {'inputs': [1], 'function': 'filter_color', 'value_inputs': ['green']}, {'inputs': [2], 'function': 'count', 'value_inputs': []}, \n {'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [4], 'function': 'filter_size', 'value_inputs': ['large']}, \n {'inputs': [5], 'function': 'filter_color', 'value_inputs': ['purple']}, {'inputs': [6], 'function': 'filter_material', 'value_inputs': ['metal']}, \n {'inputs': [7], 'function': 'filter_shape', 'value_inputs': ['cube']}, {'inputs': [8], 'function': 'count', 'value_inputs': []}, \n {'inputs': [3, 9], 'function': 'greater_than', 'value_inputs': []}], 'question_index': 0, 'image_filename': 'CLEVR_train_000000.png', 'question_family_index': 2,\n 'split': 'train', 'answer': 'yes', 'question': 'Are there more big green things than large purple shiny cubes?'}]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.load_dataset\", MagicMock( side_effect = [ dataset_content ] )):\n clevr = CLEVR(\"clevr_training\", config[\"clevr_training\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = clevr[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['image_ids'], 'CLEVR_train_000000.png')\n self.assertEqual(sample['question_type_ids'], 4)\n self.assertEqual(sample['question_type_names'], 'greater_than')\n self.assertEqual(sample['questions'], 'Are there more big green things than large purple shiny cubes?')\n self.assertEqual(sample['answers'], 'yes')\n \n \n def test_validation_set(self):\n \"\"\"\n Tests the validation split.\n\n ..note:\n Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_val_questions.json' is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"clevr_validation\": {\"split\": \"validation\", \"globals\": {\"image_height\": \"clevr_image_height\", \"image_width\": \"clevr_image_width\"}}})\n\n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'):\n\n # Create object.\n clevr = CLEVR(\"clevr_validation\", config[\"clevr_validation\"])\n \n # Check dataset size.\n self.assertEqual(len(clevr), 149991)\n\n # Get sample.\n sample = clevr[0]\n\n else: \n dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, \n {'inputs': [1], 'function': 'filter_material', 'value_inputs': ['metal']}, {'inputs': [2], 'function': 'unique', 'value_inputs': []}, \n {'inputs': [3], 'function': 'same_shape', 'value_inputs': []}, {'inputs': [4], 'function': 'exist', 'value_inputs': []}], \n 'question_index': 0, 'image_filename': 'CLEVR_val_000000.png', 'question_family_index': 39, 'split': 'val', 'answer': 'no', 'question': 'Are there any other things that are the same shape as the big metallic object?'}]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.load_dataset\", MagicMock( side_effect = [ dataset_content ] )):\n clevr = CLEVR(\"clevr_validation\", config[\"clevr_validation\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = clevr[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['image_ids'], 'CLEVR_val_000000.png')\n self.assertEqual(sample['question_type_ids'], 10)\n self.assertEqual(sample['question_type_names'], 'exist')\n self.assertEqual(sample['questions'], 'Are there any other things that are the same shape as the big metallic object?')\n self.assertEqual(sample['answers'], 'no')\n \n\n def test_test_set(self):\n \"\"\"\n Tests the test split.\n\n ..note:\n Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_test_questions.json' is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"clevr_test\": {\"split\": \"test\", \"globals\": {\"image_height\": \"clevr_image_height\", \"image_width\": \"clevr_image_width\"}}})\n \n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'):\n\n # Create object.\n clevr = CLEVR(\"clevr_test\", config[\"clevr_test\"])\n \n # Check dataset size.\n self.assertEqual(len(clevr), 149988)\n\n # Get sample.\n sample = clevr[0]\n\n else: \n dataset_content = [{'image_index': 0, 'split': 'test', 'image_filename': 'CLEVR_test_000000.png', 'question_index': 0, 'question': 'Is there anything else that is the same shape as the small brown matte object?'}]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.load_dataset\", MagicMock( side_effect = [ dataset_content ] )):\n clevr = CLEVR(\"clevr_test\", config[\"clevr_test\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.clevr.CLEVR.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = clevr[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['image_ids'], 'CLEVR_test_000000.png')\n self.assertEqual(sample['question_type_ids'], -1)\n self.assertEqual(sample['question_type_names'], '')\n self.assertEqual(sample['questions'], 'Is there anything else that is the same shape as the small brown matte object?')\n self.assertEqual(sample['answers'], '')\n \n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/components/tasks/gqa_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nfrom unittest.mock import MagicMock, patch\nfrom os import path\n\nfrom ptp.components.mixins.io import check_file_existence\nfrom ptp.components.tasks.image_text_to_class.gqa import GQA\nfrom ptp.configuration.config_interface import ConfigInterface\n\n\nclass TestGQA(unittest.TestCase):\n\n\n def test_training_0_split(self):\n \"\"\"\n Tests the training_0 split.\n\n ..note:\n Test on real data is performed only if adequate json source file is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"gqa_training_0\": {\"split\": \"training_0\", \"globals\": {\"image_height\": \"gqa_image_height\", \"image_width\": \"gqa_image_width\"}}})\n \n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/gqa/questions1.2/train_all_questions'),'train_all_questions_0.json'):\n\n # Create object.\n task = GQA(\"gqa_training_0\", config[\"gqa_training_0\"])\n \n # Check dataset size.\n self.assertEqual(len(task), 1430536)\n\n # Get sample.\n sample = task[0]\n\n else: \n processed_dataset_content = [ {'sample_ids': '07333408', 'image_ids': '2375429', 'questions': 'What is on the white wall?', 'answers': 'pipe', 'full_answers': 'The pipe is on the wall.'} ]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.load_dataset\", MagicMock( side_effect = [ processed_dataset_content ] )):\n task = GQA(\"gqa_training_0\", config[\"gqa_training_0\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = task[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['sample_ids'], '07333408')\n self.assertEqual(sample['image_ids'], '2375429')\n self.assertEqual(sample['questions'], 'What is on the white wall?')\n self.assertEqual(sample['answers'], 'pipe')\n self.assertEqual(sample['full_answers'], 'The pipe is on the wall.')\n\n\n def test_validation_split(self):\n \"\"\"\n Tests the validation split.\n\n ..note:\n Test on real data is performed only if adequate json source file is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"gqa_validation\": {\"split\": \"validation\", \"globals\": {\"image_height\": \"gqa_image_height\", \"image_width\": \"gqa_image_width\"}}})\n \n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/gqa/questions1.2'),'val_all_questions.json'):\n\n # Create object.\n task = GQA(\"gqa_validation\", config[\"gqa_validation\"])\n \n # Check dataset size.\n self.assertEqual(len(task), 2011853)\n\n # Get sample.\n sample = task[0]\n\n else: \n processed_dataset_content = [ {'sample_ids': '05451384', 'image_ids': '2382986', 'questions': 'Are there blankets under the brown cat?', 'answers': 'no', 'full_answers': 'No, there is a towel under the cat.'} ]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.load_dataset\", MagicMock( side_effect = [ processed_dataset_content ] )):\n task = GQA(\"gqa_validation\", config[\"gqa_validation\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = task[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['sample_ids'], '05451384')\n self.assertEqual(sample['image_ids'], '2382986')\n self.assertEqual(sample['questions'], 'Are there blankets under the brown cat?')\n self.assertEqual(sample['answers'], 'no')\n self.assertEqual(sample['full_answers'], 'No, there is a towel under the cat.')\n\n\n def test_test_dev_split(self):\n \"\"\"\n Tests the test_dev split.\n\n ..note:\n Test on real data is performed only if adequate json source file is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"gqa_testdev\": {\"split\": \"test_dev\", \"globals\": {\"image_height\": \"gqa_image_height\", \"image_width\": \"gqa_image_width\"}}})\n \n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/gqa/questions1.2'),'testdev_all_questions.json'):\n\n # Create object.\n task = GQA(\"gqa_testdev\", config[\"gqa_testdev\"])\n \n # Check dataset size.\n self.assertEqual(len(task), 172174)\n\n # Get sample.\n sample = task[0]\n\n else: \n processed_dataset_content = [ {'sample_ids': '20968379', 'image_ids': 'n288870', 'questions': 'Do the shorts have dark color?', 'answers': 'yes', 'full_answers': 'Yes, the shorts are dark.'} ]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.load_dataset\", MagicMock( side_effect = [ processed_dataset_content ] )):\n task = GQA(\"gqa_testdev\", config[\"gqa_testdev\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = task[0]\n\n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['sample_ids'], '20968379')\n self.assertEqual(sample['image_ids'], 'n288870')\n self.assertEqual(sample['questions'], 'Do the shorts have dark color?')\n self.assertEqual(sample['answers'], 'yes')\n self.assertEqual(sample['full_answers'], 'Yes, the shorts are dark.')\n\n\n def test_test_split(self):\n \"\"\"\n Tests the test split.\n\n ..note:\n Test on real data is performed only if adequate json source file is found.\n \"\"\"\n # Empty config.\n config = ConfigInterface()\n config.add_config_params({\"gqa_test\": {\"split\": \"test\", \"globals\": {\"image_height\": \"gqa_image_height\", \"image_width\": \"gqa_image_width\"}}})\n \n # Check the existence of test set.\n if False: #check_file_existence(path.expanduser('~/data/gqa/questions1.2'),'test_all_questions.json'):\n\n # Create object.\n task = GQA(\"gqa_test\", config[\"gqa_test\"])\n \n # Check dataset size.\n self.assertEqual(len(task), 1340048)\n\n # Get sample.\n sample = task[0]\n\n else: \n processed_dataset_content = [ {'sample_ids': '201971873', 'image_ids': 'n15740', 'questions': 'Is the blanket to the right of a pillow?', 'answers': '', 'full_answers': ''} ]\n\n # Mock up the load_dataset method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.load_dataset\", MagicMock( side_effect = [ processed_dataset_content ] )):\n task = GQA(\"gqa_test\", config[\"gqa_test\"])\n\n # Mock up the get_image method.\n with patch( \"ptp.components.tasks.image_text_to_class.gqa.GQA.get_image\", MagicMock( side_effect = [ \"0\" ] )):\n sample = task[0]\n \n # Check sample.\n self.assertEqual(sample['indices'], 0)\n self.assertEqual(sample['sample_ids'], '201971873')\n self.assertEqual(sample['image_ids'], 'n15740')\n self.assertEqual(sample['questions'], 'Is the blanket to the right of a pillow?')\n self.assertEqual(sample['answers'], '')\n self.assertEqual(sample['full_answers'], '')\n \n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/components/tasks/task_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.components.tasks.task import Task\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.configuration.config_interface import ConfigInterface\n\n\nclass MockupTask (Task):\n \"\"\"\n Mockup task class.\n \"\"\"\n def __init__(self, name, config):\n Task.__init__(self, name, None, config)\n\n def output_data_definitions(self):\n return {\n \"inputs\": DataDefinition([-1, 1], [list, int], \"inputs\"),\n \"targets\": DataDefinition([-1, -1, -1], [list, list, str], \"targets\")\n }\n\n\nclass TestTask(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestTask, self).__init__(*args, **kwargs)\n\n # Overwrite abc abstract methods.\n MockupTask.__abstractmethods__=set()\n # Create mocked-up task.\n config = ConfigInterface()\n self.task = MockupTask(\"test\", config)\n\n def test_crete_data_streams_key_present(self):\n \"\"\" Tests whether the created data dict contains required keys. \"\"\"\n data_streams = self.task.create_data_streams(1)\n # Check presence of index.\n self.assertEqual(data_streams['indices'], 1) # Even if we didn't explicitly indicated that in definitions!\n self.assertEqual(data_streams['inputs'], None)\n self.assertEqual(data_streams['targets'], None)\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/configuration/config_interface_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.configuration.config_interface import ConfigInterface\n\nclass TestConfigInterface(unittest.TestCase):\n\n def test_default_params(self):\n config = ConfigInterface()\n # Add params - first method.\n config.add_default_params({'default_0': {'default_1': 'str'}})\n self.assertNotEqual(config['default_0'], None)\n self.assertEqual(config['default_0']['default_1'], 'str')\n\n # Remove params - first method.\n config.del_default_params(['default_0', 'default_1'])\n with self.assertRaises(KeyError):\n _ = config['default_0']['default_1']\n\n # Add params - second method.\n config['default_0'].add_default_params({'default_2': 'str'})\n\n # Remove params - second method.\n config['default_0'].del_default_params('default_2')\n with self.assertRaises(KeyError):\n _ = config['default_0']['default_2']\n\n # Add 3rd parameter under 0.\n config['default_0'].add_default_params({'default_3': 'str'})\n\n # Remove the main section.\n config.del_default_params('default_0')\n with self.assertRaises(KeyError):\n _ = config['default_0']\n\n\n def test_config_params(self):\n config = ConfigInterface()\n # Add params.\n config.add_config_params({'config_0': {'config_1': 'int'}})\n self.assertNotEqual(config['config_0'], None)\n self.assertEqual(config['config_0']['config_1'], 'int')\n\n # Remove params.\n config.del_config_params(['config_0', 'config_1'])\n with self.assertRaises(KeyError):\n _ = config['config_0']['config_1']\n\n def test_overwrite_params(self):\n config = ConfigInterface()\n config.add_config_params({'under': True})\n config.add_default_params({'under': False})\n self.assertEqual(config['under'], True)\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/configuration/config_registry_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.configuration.config_registry import ConfigRegistry\n\nclass TestConfigRegistry(unittest.TestCase):\n\n def test_default_params(self):\n config = ConfigRegistry()\n # Add params.\n config.add_default_params({'default_0': {'default_1': 'str'}})\n self.assertNotEqual(config['default_0'], None)\n self.assertEqual(config['default_0']['default_1'], 'str')\n\n # Remove params.\n config.del_default_params(['default_0', 'default_1'])\n with self.assertRaises(KeyError):\n _ = config['default_0']['default_1']\n\n\n def test_config_params(self):\n config = ConfigRegistry()\n # Add params.\n config.add_config_params({'config_0': {'config_1': 'int'}})\n self.assertNotEqual(config['config_0'], None)\n self.assertEqual(config['config_0']['config_1'], 'int')\n\n # Remove params.\n config.del_config_params(['config_0', 'config_1'])\n with self.assertRaises(KeyError):\n _ = config['config_0']['config_1']\n\n\n def test_overwrite_params(self):\n config = ConfigRegistry()\n config.add_config_params({'under': True})\n config.add_default_params({'under': False})\n self.assertEqual(config['under'], True)\n\n\n#if __name__ == \"__main__\":\n# unittest.main()" }, { "path": "tests/configuration/handshaking_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.components.component import Component\nfrom ptp.data_types.data_definition import DataDefinition\nfrom ptp.configuration.config_interface import ConfigInterface\n\n\nclass MockupComponent (Component):\n \"\"\"\n Mockup component class.\n \"\"\"\n def __init__(self):\n Component.__init__(self, \"MockupComponent\", None, ConfigInterface())\n\n def input_data_definitions(self):\n return {\n \"input1\": DataDefinition([-1, 1], [list, int], \"comment1\"),\n \"input2\": DataDefinition([-1, -1, -1], [list, list, str], \"comment2\"),\n \"input3\": DataDefinition([-1, -1], [float], \"comment3\")\n }\n\n def output_data_definitions(self):\n return {\n \"output\": DataDefinition([1], [int], \"Value (scalar)\")\n }\n\n\nclass TestHandshaking(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestHandshaking, self).__init__(*args, **kwargs)\n\n # Overwrite abc abstract methods.\n MockupComponent.__abstractmethods__=set()\n # Create mockup component\n self.component = MockupComponent()\n\n\n def test_handshake_input_definitions_keys(self):\n \"\"\" Tests handskaking of input definition keys. \"\"\"\n # Both inputs missing.\n all_defs = {}\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 3)\n\n # One input missing.\n all_defs[\"input1\"] = DataDefinition([-1, 1], [list, int], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 2)\n\n # All inputs ok.\n all_defs[\"input2\"] = DataDefinition([-1, -1, -1], [list, list, str], \"comment2\")\n all_defs[\"input3\"] = DataDefinition([-1, -1], [float], \"comment3\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 0)\n\n\n def test_handshake_input_definitions_dimensions(self):\n \"\"\" Tests handskaking of input definition keys. \"\"\"\n all_defs = {}\n all_defs[\"input2\"] = DataDefinition([-1, -1, -1], [list, list, str], \"comment2\")\n all_defs[\"input3\"] = DataDefinition([-1, -1], [float], \"comment3\")\n\n # One input with wrong number of dimensions.\n all_defs[\"input1\"] = DataDefinition([-1, 10], [list, int], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 1)\n\n # One input with wrong number of dimensions, but with dynamic size provided while we expect fixed.\n all_defs[\"input1\"] = DataDefinition([-1, -1], [list, int], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 1)\n\n # One input with wrong number of dimensions, but it is the \"agnostic\" one.\n all_defs[\"input1\"] = DataDefinition([1, 1], [list, int], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 0)\n\n\n def test_handshake_input_definitions_types(self):\n \"\"\" Tests handskaking of input definition keys. \"\"\"\n all_defs = {}\n all_defs[\"input2\"] = DataDefinition([-1, -1, -1], [list, list, str], \"comment2\")\n all_defs[\"input3\"] = DataDefinition([-1, -1], [float], \"comment3\")\n\n # One input with wrong number of types.\n all_defs[\"input1\"] = DataDefinition([-1, 1], [list], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 1)\n\n # One input with one wrong type.\n all_defs[\"input1\"] = DataDefinition([-1, 1], [list, str], \"comment\")\n self.assertEqual(self.component.handshake_input_definitions( all_defs, log_errors=False ), 1)\n\n\n def test_extension_definitions(self):\n \"\"\" Tests extension of output definition keys. \"\"\"\n all_defs = {} \n\n # Key not existing in output definitions - ADD.\n all_defs[\"output2\"] = DataDefinition([-1, -1, -1], [list, list, str], \"comment\")\n self.assertEqual(self.component.export_output_definitions( all_defs, log_errors=False ), 0)\n\n # Key already existing in output definitions.\n all_defs[\"output\"] = DataDefinition([-1, -1, -1], [list, list, str], \"comment\")\n self.assertEqual(self.component.export_output_definitions( all_defs, log_errors=False ), 1)\n\n\n" }, { "path": "tests/data_types/data_definition_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.data_types.data_definition import DataDefinition\n\nclass TestDataDefinition(unittest.TestCase):\n\n def test_values(self):\n \"\"\" Tests whether the values are set. \"\"\"\n dd = DataDefinition([1], [int], \"Value\")\n\n self.assertEqual(dd.dimensions, [1])\n self.assertEqual(dd.types,[int])\n self.assertEqual(dd.description, \"Value\")\n\n\n def test_override(self):\n \"\"\" Tests whether values cannot be overriden. \"\"\"\n dd = DataDefinition([1], [int], \"Value\")\n\n with self.assertRaises(AttributeError):\n dd.dimensions = [1,2]\n with self.assertRaises(AttributeError):\n dd.types = [str,list]\n with self.assertRaises(AttributeError):\n dd.description = \"New Description\"\n\n" }, { "path": "tests/data_types/data_streams_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\n\nfrom ptp.data_types.data_streams import DataStreams\n\nclass TestDataStreams(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestDataStreams, self).__init__(*args, **kwargs)\n\n data_definitions = {\n 'inputs': None,\n 'targets': None\n }\n # Create object.\n self.data_streams = DataStreams(data_definitions)\n\n def test_keys_present(self):\n \"\"\" Tests whether the original keys are present and can be retrieved/modified. \"\"\"\n self.assertTrue('inputs' in self.data_streams.keys() )\n self.assertTrue('targets' in self.data_streams.keys() )\n\n # Check initial value.\n self.assertEqual(self.data_streams['inputs'], None)\n # Modify and retrieve.\n self.data_streams['inputs'] = 1.2\n self.assertEqual(self.data_streams['inputs'], 1.2)\n\n\n def test_keys_absent(self):\n \"\"\" Tests whether absent keys are really absent and cannot be simply added. \"\"\"\n with self.assertRaises(KeyError):\n a = self.data_streams[\"predictions\"]\n with self.assertRaises(KeyError):\n self.data_streams[\"predictions\"] = 12\n\n\n def test_keys_extend(self):\n \"\"\" Tests whether append works as expected. \"\"\"\n # Cannot add existing key.\n with self.assertRaises(KeyError):\n self.data_streams.publish( {\"inputs\": 1.5 } ) \n # Can add new key.\n self.data_streams.publish( {\"predictions\": 12 } )\n self.assertEqual(self.data_streams['predictions'], 12)\n\n" }, { "path": "tests/utils/app_state_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nfrom ptp.utils.app_state import AppState\n\nclass TestAppState(unittest.TestCase):\n\n\n def test_01keys_present(self):\n \"\"\" Tests whether the original keys are present and can be retrieved/modified. \"\"\"\n # Initialize object.\n app_state = AppState()\n # Add global.\n app_state[\"global1\"] = 1 \n # Check its value.\n self.assertEqual(app_state['global1'], 1)\n\n def test_02keys_present_singleton(self):\n \"\"\" Tests whether the original keys are still present in new AppState \"instance\". \"\"\"\n # Initialize object.\n app_state = AppState()\n # Check its value.\n self.assertEqual(app_state['global1'], 1)\n\n def test_03keys_absent(self):\n \"\"\" Tests whether absent keys are really absent. \"\"\"\n with self.assertRaises(KeyError):\n a = AppState()[\"global2\"]\n\n def test_04keys_overwrite(self):\n \"\"\" Tests whether you can overwrite existing key. \"\"\"\n with self.assertRaises(KeyError):\n AppState()[\"global1\"] = 2\n" }, { "path": "tests/utils/statistics_tests.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (C) tkornuta, IBM Corporation 2019\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n__author__ = \"Tomasz Kornuta\"\n\nimport unittest\nimport random\nimport numpy as np\n\nfrom ptp.utils.statistics_collector import StatisticsCollector\nfrom ptp.utils.statistics_aggregator import StatisticsAggregator\n\nclass TestStatistics(unittest.TestCase):\n\n def __init__(self, *args, **kwargs):\n super(TestStatistics, self).__init__(*args, **kwargs)\n\n def test_collector_string(self):\n \"\"\" Tests whether the collector is collecting and producing the right string. \"\"\"\n\n stat_col = StatisticsCollector()\n stat_col.add_statistics('loss', '{:12.10f}')\n stat_col.add_statistics('episode', '{:06d}')\n stat_col.add_statistics('acc', '{:2.3f}')\n stat_col.add_statistics('acc_help', None)\n\n # Episode 0.\n stat_col['episode'] = 0\n stat_col['loss'] = 0.7\n stat_col['acc'] = 100\n stat_col['acc_help'] = 121\n\n # Export.\n #csv_file = stat_col.initialize_csv_file('./', 'collector_test.csv')\n #stat_col.export_to_csv(csv_file)\n self.assertEqual(stat_col.export_to_string(), \"loss 0.7000000000; episode 000000; acc 100.000 \")\n\n # Episode 1.\n stat_col['episode'] = 1\n stat_col['loss'] = 0.7\n stat_col['acc'] = 99.3\n\n stat_col.add_statistics('seq_length', '{:2.0f}')\n stat_col['seq_length'] = 5\n\n # Export.\n #stat_col.export_to_csv(csv_file)\n self.assertEqual(stat_col.export_to_string('[Validation]'), \"loss 0.7000000000; episode 000001; acc 99.300; seq_length 5 [Validation]\")\n\n # Empty.\n stat_col.empty()\n self.assertEqual(stat_col.export_to_string(), \"loss ; episode ; acc ; seq_length \")\n\n def test_aggregator_string(self):\n \"\"\" Tests whether the collector is aggregating and producing the right string. \"\"\"\n\n stat_col = StatisticsCollector()\n stat_agg = StatisticsAggregator()\n\n # Add default statistics with formatting.\n stat_col.add_statistics('loss', '{:12.10f}')\n stat_col.add_statistics('episode', '{:06d}')\n stat_col.add_statistics('batch_size', None)\n\n # create some random values\n loss_values = random.sample(range(100), 100)\n # \"Collect\" basic statistics.\n for episode, loss in enumerate(loss_values):\n stat_col['episode'] = episode\n stat_col['loss'] = loss\n stat_col['batch_size'] = 1\n # print(stat_col.export_statistics_to_string())\n\n # Empty before aggregation.\n self.assertEqual(stat_agg.export_to_string(), \" \")\n\n # Number of aggregated episodes.\n stat_agg.add_aggregator('acc_mean', '{:2.5f}')\n collected_loss_values = stat_col['loss']\n batch_sizes = stat_col['batch_size']\n stat_agg['acc_mean'] = np.mean(collected_loss_values) / np.sum(batch_sizes)\n\n # Aggregated result.\n self.assertEqual(stat_agg.export_to_string('[Epoch 1]'), \"acc_mean 0.49500 [Epoch 1]\")\n\n\n#if __name__ == \"__main__\":\n# unittest.main()" } ]