Repository: braincorp/PVM Branch: master Commit: 3de2683634f3 Files: 106 Total size: 835.1 KB Directory structure: gitextract__0_tzylg/ ├── .gitmodules ├── LICENSE.TXT ├── PVM_framework/ │ ├── AbstractExecutionManager.py │ ├── AbstractExecutionUnit.py │ ├── Accelerated.cpp │ ├── Accelerated.h │ ├── CoreUtils.py │ ├── LowLevelCPUControlx86.pyx │ ├── MLP.py │ ├── PVM_Create.py │ ├── PVM_SignalProvider.py │ ├── PVM_Storage.py │ ├── PVM_datasets.py │ ├── PVM_debug_console.py │ ├── PVM_display_helper.py │ ├── PVM_options.py │ ├── PVM_tracker.json │ ├── PVM_upgrade_dict.py │ ├── SharedArray.py │ ├── Sync.cpp │ ├── Sync.h │ ├── SyncUtils.pyx │ ├── SyncUtils_python.py │ ├── __init__.py │ ├── debug_logger.py │ └── fast_routines.pyx ├── PVM_models/ │ ├── PVM_Manager.py │ ├── PVM_plot_error.py │ ├── PVM_run.py │ ├── PVM_tracker.py │ ├── PVM_unit_2step_residual_v1.py │ ├── PVM_unit_test.py │ ├── PVM_unit_v1.py │ ├── __init__.py │ ├── demo00_run.py │ ├── demo00_unit.py │ ├── demo01_run.py │ ├── demo01_unit.py │ ├── demo02_run.py │ ├── demo02_unit.py │ ├── demo03_run.py │ ├── demo03_unit.py │ ├── demo04_run.py │ ├── demo04_unit.py │ ├── model_zoo/ │ │ ├── classic_medium.json │ │ ├── experiment1_face.json │ │ ├── experiment1_green_ball.json │ │ ├── experiment1_stop_sign.json │ │ ├── experiment2.json │ │ └── very_deep.json │ ├── process_dream_data.py │ └── run_tracking_benchmark.py ├── PVM_tests/ │ ├── test_MLP.py │ ├── test_SharedArray.py │ ├── test_SyncUtils.py │ ├── test_bounding_region.py │ ├── test_create.py │ ├── test_fast_routines.py │ └── test_labeled_movie.py ├── PVM_tools/ │ ├── __init__.py │ ├── abstract_bounding_boxer.py │ ├── abstract_tracker.py │ ├── benchmark.py │ ├── bounding_region.py │ └── labeled_movie.py ├── README.md ├── add_thirdparty_tracker_submodules.sh ├── docs/ │ ├── Makefile │ ├── conf.py │ ├── generate.sh │ ├── index.rst │ └── make.bat ├── download_data.sh ├── install_local.sh ├── install_ubuntu_dependencies.sh ├── other_trackers/ │ ├── ASMSearcher.hpp │ ├── __init__.py │ ├── backprojection.py │ ├── bounding_boxer.py │ ├── center_vision_tracker.py │ ├── cmt_vision_tracker.py │ ├── color_vision_tracker.py │ ├── null_vision_tracker.py │ ├── opentld_python.cpp │ ├── setup_opentld.py │ ├── setup_struck.py │ ├── struck.cpp │ ├── struck.h │ ├── struck_bindings.pyx │ ├── struck_tracker.py │ ├── test_struck_bindings.py │ ├── test_tld_basic.py │ └── tld_vision_tracker.py ├── pytest.ini ├── requirements.txt ├── setup.py ├── test └── tracker_tools/ ├── __init__.py ├── export_to_zip.py ├── images_to_PVM_pickle.py ├── label_PVM_pickle.py ├── movie_to_PVM_pickle.py ├── play_PVM_pickle.py ├── raw_to_PVM_pickle.py ├── scale_PVM_pickle.py └── upgrade_cloud_lib.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitmodules ================================================ [submodule "other_trackers/original_struck"] path = other_trackers/original_struck url = https://github.com/gnebehay/STRUCK [submodule "other_trackers/original_cmt"] path = other_trackers/original_cmt url = https://github.com/gnebehay/CMT [submodule "other_trackers/original_opentld"] path = other_trackers/original_opentld url = https://github.com/gnebehay/OpenTLD ================================================ FILE: LICENSE.TXT ================================================ License THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. 1. Definitions "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with one or more other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. "Licensor" means the individual, individuals, entity or entities that offer(s) the Work under the terms of this License. "Original Author" means the individual, individuals, entity or entities who created the Work. "Work" means the copyrightable work of authorship offered under the terms of this License. "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. "License Elements" means the following high-level license attributes as selected by Licensor and indicated in the title of this License: Attribution, Noncommercial, ShareAlike. 2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. 3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; to create and reproduce Derivative Works provided that any such Derivative Work, including any translation in any medium, takes reasonable steps to clearly label, demarcate or otherwise identify that changes were made to the original Work. For example, a translation could be marked "The original work was translated from English to Spanish," or a modification could indicate "The original work has been modified."; to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works; The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved, including but not limited to the rights set forth in Sections 4(e) and 4(f). 4. Restrictions. The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that restrict the terms of this License or the ability of a recipient of the Work to exercise the rights granted to that recipient under the terms of the License. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. When You distribute, publicly display, publicly perform, or publicly digitally perform the Work, You may not impose any technological measures on the Work that restrict the ability of a recipient of the Work from You to exercise the rights granted to that recipient under the terms of the License. This Section 4(a) applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by Section 4(d), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by Section 4(d), as requested. You may distribute, publicly display, publicly perform, or publicly digitally perform a Derivative Work only under: (i) the terms of this License; (ii) a later version of this License with the same License Elements as this License; or, (iii) either the unported Creative Commons license or a Creative Commons license for another jurisdiction (either this or a later license version) that contains the same License Elements as this License (e.g. Attribution-NonCommercial-ShareAlike 3.0 (Unported)) ("the Applicable License"). You must include a copy of, or the Uniform Resource Identifier for, the Applicable License with every copy or phonorecord of each Derivative Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Derivative Works that restrict the terms of the Applicable License or the ability of a recipient of the Work to exercise the rights granted to that recipient under the terms of the Applicable License. You must keep intact all notices that refer to the Applicable License and to the disclaimer of warranties. When You distribute, publicly display, publicly perform, or publicly digitally perform the Derivative Work, You may not impose any technological measures on the Derivative Work that restrict the ability of a recipient of the Derivative Work from You to exercise the rights granted to that recipient under the terms of the Applicable License. This Section 4(b) applies to the Derivative Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Derivative Work itself to be made subject to the terms of the Applicable License. You may not exercise any of the rights granted to You in Section 3 above in any manner that is primarily intended for or directed toward commercial advantage or private monetary compensation. The exchange of the Work for other copyrighted works by means of digital file-sharing or otherwise shall not be considered to be intended for or directed toward commercial advantage or private monetary compensation, provided there is no payment of any monetary compensation in connection with the exchange of copyrighted works. If You distribute, publicly display, publicly perform, or publicly digitally perform the Work (as defined in Section 1 above) or any Derivative Works (as defined in Section 1 above) or Collective Works (as defined in Section 1 above), You must, unless a request has been made pursuant to Section 4(a), keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution ("Attribution Parties") in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and, consistent with Section 3(b) in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). The credit required by this Section 4(d) may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear, if a credit for all contributing authors of the Derivative Work or Collective Work appears, then as part of these credits and in a manner at least as prominent as the credits for the other contributing authors. For the avoidance of doubt, You may only use the credit required by this Section for the purpose of attribution in the manner set out above and, by exercising Your rights under this License, You may not implicitly or explicitly assert or imply any connection with, sponsorship or endorsement by the Original Author, Licensor and/or Attribution Parties, as appropriate, of You or Your use of the Work, without the separate, express prior written permission of the Original Author, Licensor and/or Attribution Parties. For the avoidance of doubt, where the Work is a musical composition: Performance Royalties Under Blanket Licenses. Licensor reserves the exclusive right to collect whether individually or, in the event that Licensor is a member of a performance rights society (e.g. ASCAP, BMI, SESAC), via that society, royalties for the public performance or public digital performance (e.g. webcast) of the Work if that performance is primarily intended for or directed toward commercial advantage or private monetary compensation. Mechanical Rights and Statutory Royalties. Licensor reserves the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions), if Your distribution of such cover version is primarily intended for or directed toward commercial advantage or private monetary compensation. Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor reserves the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions), if Your public digital performance is primarily intended for or directed toward commercial advantage or private monetary compensation. 5. Representations, Warranties and Disclaimer UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND ONLY TO THE EXTENT OF ANY RIGHTS HELD IN THE LICENSED WORK BY THE LICENSOR. THE LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MARKETABILITY, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 7. Termination This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works (as defined in Section 1 above) or Collective Works (as defined in Section 1 above) from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. 8. Miscellaneous Each time You distribute or publicly digitally perform the Work (as defined in Section 1 above) or a Collective Work (as defined in Section 1 above), the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. ================================================ FILE: PVM_framework/AbstractExecutionManager.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import abc class ExecutionManager(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod def __init__(self): pass @abc.abstractmethod def start(self): """ This will be called right before the simulation starts """ pass @abc.abstractmethod def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ pass @abc.abstractmethod def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ pass @abc.abstractmethod def running(self): """ While returning True the simulation will keep going. """ @abc.abstractmethod def finish(self): """ Called at the end of execution. """ class AbstractSignalProvider(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod def __init__(self): pass @abc.abstractmethod def start(self): """ This will be called right before the simulation starts """ pass @abc.abstractmethod def get_signal(self, name, time): """ Get the signal for the current time step if time is zero, or relative if time is non zero """ pass @abc.abstractmethod def advance(self): """ Move to the next timestep """ pass @abc.abstractmethod def finish(self): """ Called at the end of execution. """ ================================================ FILE: PVM_framework/AbstractExecutionUnit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import abc class ExecutionUnit(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods Note, this needs to be extended as a class method """ pass @abc.abstractmethod def __init__(self, parameters): pass @abc.abstractmethod def generate_missing_parameters(parameters, options): """ This method can be called to generate all the missing dictionary parameters when all the other relevant viariables are known. Needs to be extended as a static method. :return: """ pass @abc.abstractmethod def execute0(self): """ This method will do the execution. It is nescessary, but you can implement more, like execute1 and so on. Their execution will be interleaved with a barrier. :return: """ pass @abc.abstractmethod def cleanup(self): """ WIll be called before the simulation is finished. Allows to copy back any remaining state into the share memory objects :return: """ pass ================================================ FILE: PVM_framework/Accelerated.cpp ================================================ /*# ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ==================================================================================*/ #include "Accelerated.h" // Uncomment below for manual loop unrolling #define UNROLL_LOOP // Computes mult * vector . matrix.T -> result void dot_transpose(double* mult, double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result) { for(int i=0; i result void dot_transpose_simple(double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result) { for(int i=0; i result void derivative_dot(double* vector, double* vector2, int vect_shape_0, double* result) { int j=0; for(; j=0.5) { x=(2*vector[j]-1)/(1-(2*vector[j]-1)); } else { x=(2*vector[j]-1)/(1+(2*vector[j]-1)); } x=fabs(x)+1; result[j] =(1.0/(2*(x*x))) * vector2[j]; } } // Computes alpha * vector1 x vector2 + beta*matrix -> result void generalized_outer(double alpha, double * vector1, int vect1_shape, double * vector2, int vect2_shape, double beta, double* matrix, double* result) { for(int i=0; i void dot_transpose(double *mult, double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result); void dot_transpose_simple(double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result); void derivative_dot(double* vector, double* vector2, int vect_shape_0, double* result); void derivative_dot_poly(double* vector, double* vector2, int vect_shape_0, double* result); void generalized_outer(double alpha, double * vector1, int vect1_shape, double * vector2, int vect2_shape, double beta, double* matrix, double* result); void dot_sigmoid(double* vector, double* matrix, int mat_shape0, int mat_shape1, double beta, double * result, int append_bias); void dot_sigmoid_poly(double* vector, double* matrix, int mat_shape0, int mat_shape1, double beta, double * result, int append_bias); void dot_add(double* vector, double* matrix, int mat_shape0, int mat_shape1, double * result, int append_bias); void sigmoid_poly(double* result, int shape, double beta); void sigmoid(double* result, int shape, double beta); ================================================ FILE: PVM_framework/CoreUtils.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cPickle import gzip import sys import time import multiprocessing as mp import socket import threading import PVM_framework.PVM_debug_console as PVM_debug_console import os import debug_logger import logging import glob import importlib import random import datetime try: import PVM_framework.SyncUtils as SyncUtils except: import PVM_framework.SyncUtils_python as SyncUtils import pickle try: from cStringIO import StringIO except ImportError: from StringIO import StringIO # Legacy class/module renaming renametable = {'future_encoder_framework.SharedArray': 'PVM_framework.SharedArray', 'PVM_tools.labeled_movie': 'PVM_tools.labeled_movie', 'PVM_tools.bounding_region': 'PVM_tools.bounding_region', } def mapname(name): if name in renametable: return renametable[name] return name def mapped_load_global(self): module = mapname(self.readline()[:-1]) name = mapname(self.readline()[:-1]) klass = self.find_class(module, name) self.append(klass) def load_legacy_pickle(str): file = StringIO(str) unpickler = pickle.Unpickler(file) unpickler.dispatch[pickle.GLOBAL] = mapped_load_global return unpickler.load() def load_legacy_pickle_file(file): unpickler = pickle.Unpickler(file) unpickler.dispatch[pickle.GLOBAL] = mapped_load_global return unpickler.load() def save_model(pobject, filename, protocol=-1): """ Save an object to a compressed disk file. :param pobject: serializable python object :type pobject: object :param filename: string representing a path to a file :type filename: str :param protocol: optional protocol parameter (defaults to -1 which is the latest available) :type protocol: int """ logging.info("Saving a model to a file " + str(filename)) pfile = gzip.GzipFile(filename, 'wb') cPickle.dump(pobject, pfile, protocol) pfile.close() logging.info("Saved a model to a file " + str(filename)) logging.info("Warning: Saved to a python pickle. Note, if you ever use this code for production consider using other " "serializing methods. Pickle is python specific and may pose a security threat") def load_model(filename): """ Loads a compressed object from disk :param filename: string representing a path to a file :type filename: str :return: python object loaded from file :rtype: object """ logging.info("Loading a model from a file " + str(filename)) pfile = gzip.GzipFile(filename, 'rb') prop_dict = load_legacy_pickle(pfile.read()) pfile.close() logging.info("Loaded a model from a file " + str(filename)) logging.info("Warning: Loaded from a python pickle. Note, if you ever use this code for production consider using other " "serializing methods. Pickle is python specific and may pose a security threat") return prop_dict def _worker_code(prop_dict, proc_id, barrier): """ The code executed by a single worker process, instantiates and executes a subset of all the objects described in the dictionary, synchronizes with other workers and the supervisor thread. :param prop_dict: Dictionary with simulation data :param proc_id: process id provided by the parrent (not the unix pid) :param barrier: barrier object used for synchronization :return: """ try: import PVM_framework.SyncUtils as SyncUtils except: import PVM_framework.SyncUtils_python as SyncUtils # Set the CPU flags try: import PVM_framework.LowLevelCPUControlx86 as LowLevelCPUControlx86 LowLevelCPUControlx86.set_flush_denormals() # Flush denormals will collapse all floating point operation results # to zero if they are to small to fit in the normal float representation. # Although such action may slighly affect the precission of calculations # on some processors (e.g. x86) it substantially speeds up the execution except: # Unable to set the flags print "Setting flush denormals CPU flag not available" import numpy as np np.random.seed(proc_id) # Load the execution module while True: # bacause of an aparent race condition, the first load may not be succesfull, try: unit = importlib.import_module(prop_dict['execution_unit_module']) except: continue break steps = int(unit.ExecutionUnit.execution_steps()) if barrier.worker_barrier() is False: sys.exit(0) # Instantiate all the objects stages = prop_dict['stages'] objects = {} # Local objects of the worker calls = {} max_procs = prop_dict['num_proc'] for stagenum in xrange(stages): stage = 'stage%d' % stagenum total_stage_elements = prop_dict['stage%d_size' % stagenum] my_stage_elements = range(proc_id, total_stage_elements, max_procs) objects[stage] = {} calls[stage] = {} objects[stage]['my_elements'] = my_stage_elements objects[stage]['units'] = [] for i in my_stage_elements: Cunit = unit.ExecutionUnit(parameters=prop_dict[stage][i]) objects[stage]['units'].append(Cunit) calls[stage] = [] for j in range(steps): calls[stage].append([]) for (i, _) in enumerate(objects[stage]['my_elements']): method = getattr(objects[stage]['units'][i], "execute%d" % j) calls[stage][j].append(method) # The main loop quit_now = False while True: for stagenum in range(stages): stage = 'stage%d' % stagenum for j in range(steps): while prop_dict['paused'][0] == 1: time.sleep(1) for method in calls[stage][j]: method() if barrier.worker_barrier() is False: quit_now = True break if quit_now: break if quit_now: for stagenum in range(stages): stage = 'stage%d' % stagenum for (i, _) in enumerate(objects[stage]['my_elements']): objects[stage]['units'][i].cleanup() sys.exit(0) def _supervisor_run(prop_dict, control_barrier): """ This is the supervisor (parent class) code. For efficiency reasons the loop that synchronized the worker nodes is not run in the main thread. Main thread can perform actions like window display etc. in a loosely synchronized manner with the rest of the execution. [MAIN THREAD] [ SUPERVISOR ] [ WORKER 0] ... runs manager runs the --------> executes displays windows main loop. the unit adjust dict params. Syncs to code Syncs to the workers ---------> supervisor <----> and main thread :param prop_dict: dictionary with simulation memory and parameters :param control_barrier: barrier object used for synchronization with the main thread :return: """ tryonce = True while True: try: unit = importlib.import_module(prop_dict['execution_unit_module']) except: if tryonce: logging.error("Could not load module %s, going insane in infinite loop. Kill me." % prop_dict['execution_unit_module']) tryonce = False continue break logging.info("Loaded the module %s, no reason to kill me yet." % prop_dict['execution_unit_module']) steps = unit.ExecutionUnit.execution_steps() barrier = SyncUtils.Barrier(prop_dict['num_proc']) procs = [] for i in xrange(prop_dict['num_proc']): p = mp.Process(target=_worker_code, args=(prop_dict, i, barrier)) p.start() procs.append(p) logging.info("Created workers, no one started yet") sys.stdout.flush() time.sleep(1) sys.stdout.flush() barrier.parent_barrier() N = prop_dict['N'][0] control_barrier.worker_barrier() logging.info("Starting workers") while barrier.workers_running() > 0: # Main execution loop for i in range(prop_dict['stages']): for j in range(steps): barrier.resume_workers() while prop_dict['paused'][0] == 1: time.sleep(1) barrier.parent_barrier() N += 1 prop_dict['N'][0] = N running = control_barrier.worker_barrier() if not running: prop_dict['paused'][0] = 0 barrier.resume_workers() barrier.parent_barrier(quit_workers=True) logging.info("Quitting") sys.stdout.flush() break logging.info("All have finished") sys.stdout.flush() for p in procs: p.join() def _run_debug_session(socket, prop_dict): """ Starts a debugging session :param socket: Network socket on which the session is transmitted :param prop_dict: Simulation dictionary of the currently running simulation :return: """ file = socket.makefile(mode="rw") print >>file, "" print >>file, "Predictive Vision Framework version 1.0" print >>file, "(C) 2016 Brain Corporation Technical Services" print >>file, "" print >>file, "You are connected to a debug shell" print >>file, "Type 'help' for available commands, 'quit' to exit debug shell" explorer = PVM_debug_console.InteractiveDictionaryExplorer(stdin=file, stdout=file, dict=prop_dict, infilename="engine"+str(os.getpid())) explorer.cmdloop() logging.info("Ended a remote debug session") socket.close() def _monitor_debug_session(port, stop): """ Separate thread code to monitor the debug session and end it if the stop() call is true. This is necessary because the debug server might be stuck on listen. When the simulation is finished (via any external event) the thread processing the server needs to be waken up from the listen mode. The function below accomplishes that by periodically trying to connect to it. :param port: Port on which the session is running :param stop: A function returning a bool value of whether the session should be stopped :return: """ while 1: time.sleep(0.5) if stop(): S = socket.socket(socket.AF_INET, socket.SOCK_STREAM) S.connect(("localhost", port)) S.close() logging.info("Closing the server socket") break def _run_debug_server(prop_dict, port, stop): """ Sets up a debug server at a given port :param prop_dict: Simulation dictionary of the currently running simulation :param port: Port on which to setup the debug server :param stop: A function returning a bool value of whether the session should be stopped :return: """ serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) while True: try: serversock.bind(("", port)) break except: logging.error("Port %d already taken, trying to bind to port %d" % (port, port +1)) port += 1 continue serversock.listen(5) logging.info("Listening on port " + str(port) + " for debug connections") clients = [] monitor = threading.Thread(target=_monitor_debug_session, args=(port, stop)) monitor.start() while 1: clientsock, addr = serversock.accept() if stop(): logging.info("Exiting debug session") sys.stdout.flush() break logging.info("Accepted a debug connection from " + str(addr)) clients.append(threading.Thread(target=_run_debug_session, args=(clientsock, prop_dict))) clients[-1].start() monitor.join() class ModelExecution(object): """ ModelExecution is a class allowing to control the execution of a model from the main thread. It takes a dictionary and a manager and allows for two modes of operation: 1. blocking. The start call will block until the execution is complete (as decided by the manager) 2. non blocking. Allows the main thread to control the execution step by step. :param prop_dict: Dictionary containing model data :type prop_dict: dict :param manager: Manager object controlling the aspects of execution :type AbstractExecutionManager: ExecutionManager """ def __init__(self, prop_dict, manager, port=9000): self._prop_dict = prop_dict self._manager = manager self._port = port def start(self, blocking=True): """ Begins the execution of the model. If blocking is True, the call will block the main thread until the model execution is complete. Otherwise the call will return and allow for stepped execution. .. note:: If running in non blocking mode, the call will return, but all the necessary execution threads will be running. Since the system uses busy synchronization (Spin Locks) the CPU utilization after the call will be maximal. In order to maximize machine utilization, step() should be called as fast as possible. :Example: :: manager = Manager(dict, 1000) executor = CoreUtils.ModelExecution(prop_dict=dict, manager=manager) executor.start(blocking=True) CoreUtils.save_model(dict, filename) print("Saving and running again in non blocking mode") executor.start(blocking=False) while manager.running(): executor.step() executor.finish() CoreUtils.save_model(dict, filename) :param blocking: Flag determining the execution mode, True by default :type blocking: bool """ self._logger = debug_logger.DebugLogger(self._prop_dict) self._parent_control_barrier = SyncUtils.Barrier(1, timeout=0.0001, UseSpinLock=True) self._parent_proc = mp.Process(target=_supervisor_run, args=(self._prop_dict, self._parent_control_barrier)) self._parent_proc.start() self._parent_control_barrier.parent_barrier() self._manager.start() self._finished=False self._stop_server = False self._t = threading.Thread(target=_run_debug_server, args=(self._prop_dict, self._port, lambda: self._stop_server)) self._t.start() if blocking: while self._manager.running(): self.step() self.finish() def step(self): """ Executes a single step of simulation """ # Quick turnover, data here is consistent self._logger.process_hookups() self._manager.fast_action() self._parent_control_barrier.resume_workers() # Workers execute # Longer turnover, data here might be inconsistent self._manager.slow_action() self._parent_control_barrier.parent_barrier(quit_workers=not self._manager.running()) def finish(self): """ After a session has been completed, finish removes all worker threads and closes debug server. """ if not self._finished: logging.info("Attempting to stop threads") self._stop_server = True logging.info("Attempting to join parent") self._parent_proc.join() logging.info("attempting to join thread") self._t.join() self._manager.finish() self._finished=True def run_model(prop_dict, manager, port=9000): """ Simplified way of running a model. :param prop_dict: model dictionary :param manager: execution manager :param port: port for setting up a debugging server :return: """ executor = ModelExecution(prop_dict=prop_dict, manager=manager, port=port) executor.start(blocking=True) executor.finish() ================================================ FILE: PVM_framework/LowLevelCPUControlx86.pyx ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== cdef extern from "xmmintrin.h": void _mm_setcsr(unsigned int) unsigned int _mm_getcsr() def set_flush_denormals(): """ This call will modify the Control Status Register (CSR) to instruct the CPU to flush denormals. A denormal representation of a float is such that is smaller then the smallest actual representation, with significand grater or equal to one say e.g.: 1.0000000xE-50 With the maximal negative exponent, this looks like the smallest representable number but it is not, since one can increase the precission by decreasing the value of significand: 0.0000001xE-50 is a few orders of magnitude smaller, but it is not a "normal" floating point represenation. Such denormal representations may appear form arithmetic operations, some architectures would flush such small numbers to zero. However x86 will by default keep them and perform further arithmetic operations, at the cost of very significant slowdown. Such slowdown may cause more harm (particularly in a parallel system) than the benefit of increased precision. This extension will only build on x86. Flush to zero is default on ARM architecture. """ _mm_setcsr((_mm_getcsr() & ~0x0040) | (0x0040)) ================================================ FILE: PVM_framework/MLP.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import abc import PVM_framework.fast_routines as fast_routines import PVM_framework.SharedArray as SharedArray import time try: @profile def test(): pass except: def profile(fun): return fun def random_ortho(n): """ Generates a random orthogonal matrix n x n :param n: dimesion :type n: int """ A = np.mat(np.random.random((n, n))) Q, R = np.linalg.qr(A) return Q def random_with_singular_values(m, n, singular_values): """ Generates a random matrix with given singular values :param m: first dimension of the array :type m: int :param n: second dimension of the array :type n: int :param singular_values: value of the singuar values of the generated matrix :type singular_values: float :return: matrix :rtype: np.ndarray """ Q = random_ortho(m) singular_values=np.array(singular_values) svs = singular_values.shape[0] if svs < max(n, m): singular_values = np.concatenate((singular_values, np.array([0] * (max(n, m)-svs)))) D = np.diag(singular_values) V = random_ortho(n) M = Q*D[:m, :n]*V return np.array(M) def optimal_weight_initialization(m, n, singular_value=1.1): """ Optimal weight matrix initialization """ return random_with_singular_values(m, n, [singular_value] * min(m, n)) class MLPPrototype(object): """ MLPPrototype is an abstract class to be extended by particular implementations of the multi layer perceptron. """ def __init__(self, state): pass @abc.abstractmethod def train(self, inputs, targets): """ Pass and train one vector consisting of: * inputs - 1 dimensional numpy array * targets - 1 dimensional numpy array returns the value of the perceptron as evaluated on inputs """ @abc.abstractmethod def evaluate(self, inputs): """ Returns the evaluation of a perceptron on inputs (1-d numpy array) """ @abc.abstractmethod def copy_weights(self): """ Assures that the weight matrix is stored back in the state dictionary """ @abc.abstractmethod def get_hidden(self, layer): """ Returns the activations of the hidden layer """ @abc.abstractmethod def get_deltas(self, layer): """ Returns the deltas computed by the backprop algorithm in the previous call of train. """ @abc.abstractmethod def copy_state(self): """ Copies all the state values into the state that appear in the paramaters dictionary. If the implementation already uses these arrays, the method can just pass """ def view_as_ndarray(object): """ This function takes a nested collection of lists/dictionaries with leafs that are numpy array like objects (e.g. shared numpy array) and casts them to a numpy array object which may accelerate the access time. :param object: nested collection of dictionaries/lists with leafs that are numpy array like objects. .. note:: If the leafs of the array are shared memory objects, running this method may render them non serializable, in other words, they will serialize as regular numpy arrays. """ if type(object) is list: retval = [] for element in object: retval.append(view_as_ndarray(element)) return retval if type(object) is dict: retval = {} for element in object.keys(): retval[element] = view_as_ndarray(object[element]) return retval if object is None: return None try: retval = object.view(np.ndarray) return retval except: raise Exception("Object neither a dict, nor a list and not an ndarray like") def get_layers(dims=None): """ Instantiate MLP layers with given dimensions :param dims: list of ints :return: layer dictinary """ if not dims: dims = [] layers = [] for d in dims: layers.append({'activation': SharedArray.SharedNumpyArray(d, np.float), 'error': SharedArray.SharedNumpyArray(d, np.float), 'delta': SharedArray.SharedNumpyArray(d, np.float) }) return layers def get_weights(layers=None): """ Instantiate MLP weights :param layers: layer dictionary :return: list of weight matrices """ if not layers: layers = [] mlp_weights = [] for l in range(len(layers)-1): l0 = layers[l]['activation'].shape[0] l1 = layers[l+1]['activation'].shape[0] - 1 mlp_weights.append(initialize_weights(SharedArray.SharedNumpyArray((l0, l1), np.float))) return mlp_weights def initialize_weights(w, method=0): """ Initialize a weight matrix. Two methods are supported: method 0 is the classical initialization with uniform random variable around zero with variance inversely proportional to the first matrix dimension method 1 creates a random matrix with all singular values equal to 1.1. :param w: array to be populated with values :type w: numpy.ndarray :param method: method to be used :type method: int :return: referece to the populated array :rtype: numpy.ndarray """ if method == 1: w[:] = optimal_weight_initialization(w.shape[0], w.shape[1]) elif method == 0: pass w[:] = (np.random.rand(w.shape[0], w.shape[1])-0.5)*2/np.sqrt(w.shape[0]-1) else: raise("Unknown initialization method!") return w class MLP(MLPPrototype): """ Multi Layer Perceptron with arbitrary number of layers. This sample code implements the MLPPrototype abstract class using pure Python and numpy. A distinctive feature of this code is that a lot of the data structures containing the state of the algorithm (e.g., weight matrices) are supplied externally to the constructor. The constructor then uses references to these arrays are internal state objects. This allows this code to be used in a parallel system, where state variables can be instantiated as shared memory arrays, and inspected, saved or modified externally. :param state: dictionary containing predefined fields. The fields describe the network structure and provide allocated arrays (numpy arrays) to store the calculated weight values and other data. :type state: dict This design allows the object to have external side effects, particularly side effects in a different process if supplied data strucutres are shared. The dictionary should contain: state['layers'] - list of layers each layer being a dictionary with three keys: * activation - a numpy array like object for storing activations (including the bias activation always equal to 1, being the last element) * error - a numpy array like object of the same shape as activation for storing the error * delta - a numpy array like object of the same shape as activation for storing the delta factors state['beta'] - numpy array like object containing a scaling factor for the sigmoid state['momentum'] - numpy array like object containing a momentum term for learning state['learning_rate'] - numpy array like object containing the learning rate state['mse'] - a single element numpy array that will be populated with the calculated mean squared error (MSE) state['weights'] - a list of weight matrices with compatible shapes. The shapes of weights should be: * weights[0] - weights from layer 0 to layer 1. The shape should be A_l0 x (A_l1 - 1) where A_lX stands for the shape of activation array in layer X * weights[1] - weights from layer 1 to layer 2. The shape should be A_l1 x (A_l2 - 1) * weights[2] - weights from layer 2 to layer 3. The shape should be A_l2 x (A_l3 - 1) * and so on :Example: :: X = np.array([[0, 1], [1, 1], [1, 0], [0, 0]]) Y = np.array([[1, 0], [0, 0], [0, 1], [1, 1]]) state = {} state['layers'] = [ {'activation': np.zeros((3,)), 'error': np.zeros((3,)), 'delta': np.zeros((3,))}, {'activation': np.zeros((5,)), 'error': np.zeros((5,)), 'delta': np.zeros((5,))}, {'activation': np.zeros((3,)), 'error': np.zeros((3,)), 'delta': np.zeros((3,))}, ] state['weights'] = [ MLP2.initialize_weights(np.zeros((3, 4)), False), MLP2.initialize_weights(np.zeros((5, 2)), False) ] state['beta'] = np.array([1.0]) state['learning_rate'] = np.array([0.1]) state['momentum'] = np.array([0.5]) state['mse'] = np.array([0.0]) M = MLP2(state) for n in xrange(20000): i = np.random.randint(low=0, high=4) M.train(X[i], Y[i]) for i in range(4): O = M.evaluate(X[i]) assert(np.allclose(Y[i], O, atol=0.05)) """ def copy_weights(self): pass def get_hidden(self, layer): pass def __init__(self, state): # Set up network size self.layers = view_as_ndarray(state['layers']) for layer in self.layers: assert 'activation' in layer.keys() assert 'error' in layer.keys() assert 'delta' in layer.keys() self.n_layers = len(self.layers) self.beta = state['beta'] self.momentum = state['momentum'].view(np.ndarray) self.learning_rate = state['learning_rate'].view(np.ndarray) # Initialize network self.weights = view_as_ndarray(state['weights']) self.delta_w = [] self.__prev_update = [] for i in range(len(self.weights)): assert(self.layers[i]['activation'].shape[0] == self.weights[i].shape[0]) if i > 0: assert(self.layers[i]['activation'].shape[0] >= self.weights[i-1].shape[1]+1) self.delta_w.append(np.zeros_like(self.weights[i])) self.__prev_update.append(np.zeros_like(self.weights[i])) self.mse = state['mse'].view(np.ndarray) for layer in self.layers: if "no_bias" in state.keys(): layer['activation'][-1] = 0 else: layer['activation'][-1] = 1 self.polynomial = False if "polynomial" in state: self.polynomial = True def train(self, inputs, targets, eta=None): """ Train one sample of data :param inputs: one dimensional numpy array containing the input vector :type inputs: np.ndarray :param targets: one dimesinonal numpy array containing the desired output :type targets: np.ndarray :param eta : override the learning rate :type eta: float :return: actual activation of the output layer :rtype: np.ndarray """ if eta is None: eta = self.learning_rate[0] self.mlp_forward(inputs) outputs = self.layers[-1]['activation'][:-1] self.layers[-1]['error'][:-1] = targets - outputs if not self.polynomial: fast_routines.derivative_dot_c(outputs, self.layers[-1]['error'], self.layers[-1]['delta']) else: fast_routines.derivative_dot_c_poly(outputs, self.layers[-1]['error'], self.layers[-1]['delta']) self.mlp_backpropagate() self.calculate_weight_update() self.update_weights() return outputs.copy() def evaluate(self, inputs): """ Evaluate the perceptron on a sample without learning :param inputs: one dimensional numpy array of inputs :type inputs: numpy.ndarray :return: activation of the output layer :rtype: np.ndarray """ self.mlp_forward(inputs) return self.layers[-1]['activation'][:-1].copy() def train2(self, error, eta=None): """ Similar to train only here the user supplies the error which allows for a little bit more flexibility (e.g. part of the error be a direct function of the results) :param error: one dimensional numpy array of error :type error: numpy.ndarray :param eta: override the learning rate :type eta: float :return: numpy.ndarray :rtype: numpy.ndarray """ if eta is None: eta = self.learning_rate[0] self.layers[-1]['error'][:] = 0 self.layers[-1]['error'][:error.shape[0]] = error if not self.polynomial: fast_routines.derivative_dot_c(self.layers[-1]['activation'], self.layers[-1]['error'], self.layers[-1]['delta']) else: fast_routines.derivative_dot_c_poly(self.layers[-1]['activation'], self.layers[-1]['error'], self.layers[-1]['delta']) self.mlp_backpropagate() self.calculate_weight_update() self.update_weights() def mlp_forward(self, inputs=None): """ Run the network forward. This method should not be used externally. :param inputs: one dimensional numpy array of inputs :type inputs: numpy.ndarray """ if inputs is not None: self.layers[0]['activation'][:-1] = inputs for layer in xrange(self.n_layers-1): self.mlp_forward_layer(layer) def mlp_forward_layer(self, layer): """ Propagate activations forward from layer to layer + 1 :param layer: index of the layer to propagate from :type layer: int :return: """ input = self.layers[layer]['activation'] weights = self.weights[layer] beta = self.beta[0] result = self.layers[layer+1]['activation'] if not self.polynomial: fast_routines.dot_sigmoid_c(input, weights, beta, result, 0) else: fast_routines.dot_sigmoid_c_poly(input, weights, beta, result, 0) def mlp_backpropagate_layer(self, layer): """ Propagate error backwards from layer + 1 to layer :param layer: index of the layer to propagate the error to :type layer: int :return: """ deltas = self.layers[layer]['delta'] upper_deltas = self.layers[layer+1]['delta'] activation = self.layers[layer]['activation'] weights = self.weights[layer] error = self.layers[layer]['error'] fast_routines.dot_transpose_c(upper_deltas[:weights.shape[1]], weights, error) if not self.polynomial: fast_routines.derivative_dot_c(activation, error, deltas) else: fast_routines.derivative_dot_c_poly(activation, error, deltas) def mlp_backpropagate(self): """ Propagate the error backwards from the output to the input layer :return: """ for layer in xrange(self.n_layers-2, -1, -1): self.mlp_backpropagate_layer(layer) def calculate_weight_update_layer(self, layer, eta=None): """ Given the backpropagated error and delta factors, calculate the weight update :param layer: :return: """ if eta is None: eta = self.learning_rate[0] inputs = self.layers[layer]['activation'] deltas = self.layers[layer+1]['delta'] fast_routines.generalized_outer_c(eta, inputs, deltas[:self.weights[layer].shape[1]], self.momentum[0], self.__prev_update[layer], self.delta_w[layer]) def calculate_weight_update(self, eta=None): """ Calculate weight updates for all the layers. :return """ for layer in xrange(len(self.weights)): self.calculate_weight_update_layer(layer, eta=eta) def update_weights_layer(self, layer): """ Perform the actual weight update for a layer of weights. Note layer here is the index of the input layer for the weight matrix that will be modified. :param layer: layer index :type layer: int :return: """ self.weights[layer] += self.delta_w[layer] self.__prev_update[layer][:] = self.delta_w[layer][:] def update_weights(self): """ Update weights in all layers. :return: """ for layer in xrange(len(self.weights)): self.update_weights_layer(layer) def get_activation(self, layer=0): """ Get the activation of the layers (updated when self.train or self.evaluate was called) :param layer: index of the layer (0 - input, -1 output) :type layer: int :return: activation of the hidden layer :rtype: numpy.ndarray """ return self.layers[layer]['activation'][:-1] def get_deltas(self, layer=0): """ Get the deltas calculated by backprop on hidden layers (updated when self.train was called) :param layer : index of the layer :type layer: int :return: backpropagated error for every unit in the hidden layer :rtype: numpy.ndarray """ return self.layers[layer]['delta'] def copy_state(self): """ An empty method, since all of the state is actually kept in externally supplied data structures. :return: No return value """ pass if __name__ == "__main__": # Timing with and without flushing denormals X = np.random.rand(100, 100) Y = np.random.rand(100, 100) state = {} state['layers'] = [ {'activation': np.zeros((101,)), 'error': np.zeros((101,)), 'delta': np.zeros((101,))}, {'activation': np.zeros((51,)), 'error': np.zeros((51,)), 'delta': np.zeros((51,))}, {'activation': np.zeros((101,)), 'error': np.zeros((101,)), 'delta': np.zeros((101,))}, ] state['weights'] = [ MLP.initialize_weights(np.zeros((101, 50)), False), MLP.initialize_weights(np.zeros((51, 100)), False) ] state['beta'] = np.array([1.0]) state['learning_rate'] = np.array([0.1]) state['momentum'] = np.array([1.0]) state['mse'] = np.array([0.0]) M = MLP(state) t_start = time.clock() try: import LowLevelCPUControlx86 LowLevelCPUControlx86.set_flush_denormals() except: # Unable to set the flags print "Setting flush denormals CPU flag not available" for n in xrange(100000): i = n % 100 M.train(X[i], Y[i]) t_exec = time.clock()-t_start print "Execution time %f" % t_exec ================================================ FILE: PVM_framework/PVM_Create.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import numpy as np import PVM_framework.SharedArray as SharedArray import logging import importlib import random import datetime import glob PVM_FLAG_CTRL_DREAM = 0 PVM_FLAG_CTRL_DREAM_EXP = 1 PVM_FLAG_CTRL_LEARNING = 2 PVM_FLAG_VAL_TRIGGER = 1 PVM_FLAG_VAL_CANCEL = 2 PVM_FLAG_VAL_RESET = 0 PVM_FLAG_TRIGGER_DISPLAY = 15 PVM_FLAG_TRIGGER_RECORD = 16 PVM_MAX_LAYERS = 20 # Arbitrarily set to 20, could be anything really PVM_LOG_ERROR_EVERY = 1000 # Interval for error logging PVM_FLAG_VAL_DREAM = 1 PVM_FLAG_VAL_BLINDSPOT = 2 PVM_FLAG_VAL_GRAY = 3 PVM_FLAG_VAL_NOISE = 4 PVM_FLAG_VAL_NOISE_SPOT = 5 PVM_FLAG_VAL_INV_BLINDSPOT = 6 PVM_FLAG_VAL_INV_NOISE_SPOT = 10 PVM_FLAG_VAL_DREAM_SPOT = 11 PVM_FLAG_VAL_INV_DREAM_SPOT = 7 PVM_FLAG_VAL_DEEP_DREAM = 12 PVM_FLAG_VAL_BLINKS = 8 PVM_FLAG_VAL_NOISY_SIGNAL = 9 PVM_LEARNING_FLAG = 2 PVM_LEARNING_RESET = 0 PVM_LEARNING_FREEZE = 1 PVM_LEARNING_UNFREEZE = 2 PVM_PAUSE = 1 PVM_RESUME = 0 PVM_DUMP = 2 def create_blank_dictionary(name="default", description="default", save_sources=False): """ Creates a minimal model dictionary. :return: model dict :rtype: dict """ import PVM_framework.SharedArray as SharedArray import numpy as np simulation_dict = {} simulation_dict['N'] = SharedArray.SharedNumpyArray((1,), np.int64) simulation_dict['N'][0] = 0 simulation_dict['record_filename'] = SharedArray.SharedNumpyArray((256,), np.uint8) simulation_dict['paused'] = SharedArray.SharedNumpyArray((1,), np.int) simulation_dict['paused'][0] = 0 simulation_dict['finished'] = SharedArray.SharedNumpyArray((1,), np.int) simulation_dict['finished'][0] = 0 simulation_dict['flags'] = SharedArray.SharedNumpyArray((16,), np.uint8) for i in range(16): simulation_dict['flags'][i] = 0 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['debug_infrastructure'] = {} simulation_dict['debug_infrastructure']['enabled_hookups'] = {} simulation_dict['debug_infrastructure']['disabled_hookups'] = {} logging.info("Created a blank dictionary") simulation_dict['description'] = description simulation_dict['name'] = name simulation_dict['sources'] = {} if save_sources: for filename in glob.glob("*.py"): f = open(filename, "r") simulation_dict['sources'][filename] = f.read() f.close() logging.info("Saved source file %s into the dictionary" % filename) for filename in glob.glob("*.pyx"): f = open(filename, "r") simulation_dict['sources'][filename] = f.read() f.close() logging.info("Saved source file %s into the dictionary" % filename) simulation_dict['hash'] = "%08x" % random.getrandbits(32) simulation_dict['timestamp'] = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') logging.info("Assigned hash %s to this simulation instance" % simulation_dict['hash']) logging.info("Assigned timestamp %s to this simulation instance" % simulation_dict['timestamp']) return simulation_dict def get_surround(xy, dim_x=10, dim_y=10, radius=1, exclude_self=True): """ Returns the indices of elements on the grid that are within square radius of the given xy radius = 1: 0 1 0 1 1 1 0 1 0 radius = 1.5: 1 1 1 1 1 1 1 1 1 radius = 2 0 0 1 0 0 0 1 1 1 0 1 1 1 1 1 0 1 1 1 0 0 0 1 0 0 Setting exclude_self to True removes the center unit """ laterals = [] for dx in range(-int(radius), int(radius)+1, 1): for dy in range(-int(radius), int(radius)+1, 1): if dx**2 + dy**2 > radius**2: continue if (xy[0]+dx >= 0) and (xy[0]+dx < dim_x) and (xy[1]+dy >= 0) and (xy[1]+dy < dim_y): if not (exclude_self and dx == 0 and dy == 0): laterals.append((xy[0]+dx, xy[1]+dy)) return laterals def get_fan_in(xy=(0, 0), dim_x_l=10, dim_y_l=10, dim_x_u=9, dim_y_u=9, block_x=2, block_y=2, radius=2): """ Selects a block_x x block_y subsquare in the underlying layers lying directly below the unit in the upper layer. Selects units within radius in that block. e.g. block_x=2, block_y=2 radius=2 1 1 1 1 e.g. block_x=3, block_y=3 radius=2 1 1 1 1 1 1 1 1 1 e.g. block_x=3, block_y=3 radius=1 0 1 0 1 1 1 0 1 0 """ x = xy[0] y = xy[1] if dim_x_u > 1: factor_x = ((dim_x_l-1)-(block_x-1))/(1.0*(dim_x_u-1)) else: factor_x = ((dim_x_l-1)-(block_x))/2.0 if dim_y_u > 1: factor_y = ((dim_y_l-1)-(block_y-1))/(1.0*(dim_y_u-1)) else: factor_y = ((dim_y_l-1)-(block_y))/2.0 results = [] if dim_x_u > 1 and dim_y_u > 1: for xx in range(block_x): for yy in range(block_y): if (xx-(block_x-1)*0.5)**2 + (yy-(block_y-1)*0.5)**2 > radius**2: continue results.append((int((factor_x*(x))+xx), int((factor_y*(y))+yy))) return results elif dim_x_u == 1 and dim_y_u > 1: for xx in range(block_x): for yy in range(block_y): if (xx-(block_x-1)*0.5)**2 + (yy-(block_y-1)*0.5)**2 > radius**2: continue results.append((int((dim_x_l-block_x)/2.0+xx), int((factor_y*(y)+yy)))) return results elif dim_x_u > 1 and dim_y_u == 1: for xx in range(block_x): for yy in range(block_y): if (xx-(block_x-1)*0.5)**2 + (yy-(block_y-1)*0.5)**2 > radius**2: continue results.append((int((factor_x*(x)+xx)), int((dim_y_l-block_y)/2.0+yy))) return results elif dim_x_u == 1 and dim_y_u == 1: for xx in range(block_x): for yy in range(block_y): if (xx-(block_x-1)*0.5)**2 + (yy-(block_y-1)*0.5)**2 > radius**2: continue results.append((int((dim_x_l-block_x)/2.0+xx), int((dim_y_l-block_y)/2.0+yy))) return results def connect_forward_and_back(simulation_dict, (index0, blocks_per_dim0, predicted_array), (index1, blocks_per_dim1), square_size, radius, context_factor): """ Connect two layers with a given fan-in as defined by the square_size and radius. The forward connections are accompanied by a feedback context connections back to the originating source unit. :param simulation_dict: :param square_size: :param radius: :param context_factor: :return: """ hidden_size = simulation_dict['hidden_size'] dx = hidden_size dy = hidden_size logging.info("Connecting from index %d to index %d" % (index0, index1)) logging.info("Input layer size is %d, receiving layer size is %d" % (blocks_per_dim0, blocks_per_dim1)) logging.info("Radius of connectivity %d" % radius) for x in range(blocks_per_dim1): for y in range(blocks_per_dim1): surround = get_fan_in((x, y), dim_x_l=blocks_per_dim0, dim_y_l=blocks_per_dim0, dim_x_u=blocks_per_dim1, dim_y_u=blocks_per_dim1, block_x=square_size, block_y=square_size, radius=radius) dest = index1 + x * (blocks_per_dim1) + y # destination unit for xy in surround: source = index0 + xy[0] * blocks_per_dim0 + xy[1] # source unit # Prepare the input and corresponding delta block at source input_block = simulation_dict['stage0'][source]['output_block'] delta_block = SharedArray.SharedNumpyArray_like(input_block) simulation_dict['stage0'][source]['delta_blocks'].append(delta_block) # Prepare the context and corresonding delta block at destination context_block = simulation_dict['stage0'][dest]['output_block'] delta_block2 = SharedArray.SharedNumpyArray_like(context_block) simulation_dict['stage0'][dest]['delta_blocks'].append(delta_block2) # Connect the context block to the source simulation_dict['stage0'][source]['context_blocks'].append((context_block, delta_block2, context_factor)) # Prepare the predicted blocks xx = xy[0]*hidden_size yy = xy[1]*hidden_size assert(predicted_array[xx:xx+dx, yy:yy+dy].shape == context_block.shape) predicted_block = SharedArray.DynamicView(predicted_array)[xx:xx+dx, yy:yy+dy] if not (predicted_block.shape == (dx, dy)): print predicted_block.shape raise # Connect the input to the destination together with its predicted blocks and so on. past_block = SharedArray.SharedNumpyArray_like(input_block) derivative_block = SharedArray.SharedNumpyArray_like(input_block) integral_block = SharedArray.SharedNumpyArray_like(input_block) pred_block_local = SharedArray.SharedNumpyArray_like(input_block) simulation_dict['stage0'][dest]['signal_blocks'].append((input_block, delta_block, predicted_block, past_block, derivative_block, integral_block, pred_block_local)) def connect_forward_and_back_v1(simulation_dict, (index0, blocks_per_dim0, predicted_array, predicted_array_t2), (index1, blocks_per_dim1), square_size, radius, context_factor): """ Connect two layers with a given fan in as defined by the square_size and radius. The forward connections are accompanied by feedback context connections back to the originating source unit. :param simulation_dict: :param square_size: :param radius: :param context_factor: :return: """ hidden_size = simulation_dict['hidden_size'] dx = hidden_size dy = hidden_size logging.info("Connecting from index %d to index %d" % (index0, index1)) logging.info("Input layer size is %d, receiving layer size is %d" % (blocks_per_dim0, blocks_per_dim1)) logging.info("Radius of connectivity %d" % radius) for x in range(blocks_per_dim1): for y in range(blocks_per_dim1): surround = get_fan_in((x, y), dim_x_l=blocks_per_dim0, dim_y_l=blocks_per_dim0, dim_x_u=blocks_per_dim1, dim_y_u=blocks_per_dim1, block_x=square_size, block_y=square_size, radius=radius) dest = index1 + x * (blocks_per_dim1) + y # destination unit for xy in surround: source = index0 + xy[0] * blocks_per_dim0 + xy[1] # source unit # Prepare the input and corresponding delta block at source input_block = simulation_dict['stage0'][source]['output_block'] delta_block = SharedArray.SharedNumpyArray_like(input_block) simulation_dict['stage0'][source]['delta_blocks'].append(delta_block) # Prepare the context and corresonding delta block at destination context_block = simulation_dict['stage0'][dest]['output_block'] delta_block2 = SharedArray.SharedNumpyArray_like(context_block) simulation_dict['stage0'][dest]['delta_blocks'].append(delta_block2) # Connect the context block to the source simulation_dict['stage0'][source]['context_blocks'].append((context_block, delta_block2, context_factor)) # Prepare the predicted blocks xx = xy[0]*hidden_size yy = xy[1]*hidden_size assert(predicted_array[xx:xx+dx, yy:yy+dy].shape == context_block.shape) predicted_block = SharedArray.DynamicView(predicted_array)[xx:xx+dx, yy:yy+dy] predicted_block2 = SharedArray.DynamicView(predicted_array_t2)[xx:xx+dx, yy:yy+dy] if not (predicted_block.shape == (dx, dy)): print predicted_block.shape raise # Connect the input to the destination together with its predicted blocks and so on. simulation_dict['stage0'][dest]['signal_blocks'].append((input_block, delta_block, predicted_block, predicted_block2)) def connect_back(simulation_dict, (index_from, blocks_per_dim_from), (index_to, blocks_per_dim_to), square_size, radius, context_factor): """ Connect feedback only. This function is used to connect two layers that are not directly connected (in which case the feedback connection would have been established along with the feedforward connection), e.g. in cases when feedback is sent from some layer way above to some lower layer. :param simulation_dict: :param square_size: :param radius: :param context_factor: :return: """ logging.info("Connecting back additional context from index %d to index %d" % (index_from, index_to)) logging.info("Connecting back additional context from layer size is %d, receiving layer size is %d" % (blocks_per_dim_from, blocks_per_dim_to)) logging.info("Radius of connectivity %d" % radius) for x in range(blocks_per_dim_from): for y in range(blocks_per_dim_from): surround = get_fan_in((x, y), dim_x_l=blocks_per_dim_to, dim_y_l=blocks_per_dim_to, dim_x_u=blocks_per_dim_from, dim_y_u=blocks_per_dim_from, block_x=square_size, block_y=square_size, radius=radius) source = index_from + x * (blocks_per_dim_from) + y # unit in the higher layer for xy in surround: dest = index_to + xy[0] * blocks_per_dim_to + xy[1] # unit in the lower layer context_block = simulation_dict['stage0'][source]['output_block'] delta_block2 = SharedArray.SharedNumpyArray_like(context_block) simulation_dict['stage0'][source]['delta_blocks'].append(delta_block2) # Connect the context block to the source simulation_dict['stage0'][dest]['context_blocks'].append((context_block, delta_block2, context_factor)) def gather_surround(simulation_dict, (index0, blocks_per_dim0), radius, context_factor, exclude_self=True): for x in range(blocks_per_dim0): for y in range(blocks_per_dim0): surround = get_surround((x, y), dim_x=blocks_per_dim0, dim_y=blocks_per_dim0, radius=radius, exclude_self=exclude_self) dest = index0 + x * blocks_per_dim0 + y # destination unit for xy in surround: source = xy[0] * blocks_per_dim0 + xy[1] # source unit context_block = simulation_dict['stage0'][source]['output_block'] delta_block = SharedArray.SharedNumpyArray_like(context_block) simulation_dict['stage0'][source]['delta_blocks'].append(delta_block) simulation_dict['stage0'][dest]['context_blocks'].append((context_block, delta_block, context_factor)) def create_basic_unit_v1(learning_rate, momentum, tau, readout_learning_rate): unit_parameters = dict() unit_parameters['tau'] = tau unit_parameters['primary_learning_rate'] = learning_rate unit_parameters['readout_learning_rate'] = readout_learning_rate unit_parameters['momentum'] = momentum unit_parameters['signal_blocks'] = [] unit_parameters['readout_blocks'] = [] unit_parameters['predicted_blocks'] = [] unit_parameters['delta_blocks'] = [] unit_parameters['context_blocks'] = [] return unit_parameters def generate_dict_options(name, description, options): if options['version_major'] == "1" and options["version_minor"] == "0": dic_ = generate_v1(name=name, description=description, options=options ) return dic_ def generate_v1(name, description, options): input_block_size = int(options["input_block_size"]) hidden_size = int(options["hidden_block_size"]) layer_shape = map(lambda x: int(x), options["layer_shapes"]) readout_block_size = map(lambda x: int(x), options["readout_block_size"]) readout_layer = map(lambda x: x == "1", options["enable_readout"]) lateral_radius = float(options["lateral_radius"]) fan_in_square_size = int(options["fan_in_square_size"]) fan_in_radius = int(options["fan_in_radius"]) readout_depth = int(options["readout_depth"]) ex_module = options["ex_module"] exclude_self = (options["context_exclude_self"] == '1') last_layer_context_to_all = (options["last_layer_context_to_all"] == '1') send_context_two_layers_back = (options["send_context_two_layers_back"] == '1') simulation_dict = create_blank_dictionary(name=name, description=description, save_sources=(options["save_source_files"] == '1')) simulation_dict['stages'] = 1 simulation_dict['num_proc'] = 2*mp.cpu_count()/3 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = ex_module simulation_dict['version_major'] = 1 simulation_dict['version_minor'] = 0 unit = importlib.import_module(simulation_dict['execution_unit_module']) blocks_per_dim = layer_shape layers = len(blocks_per_dim) error_log = SharedArray.SharedNumpyArray((layers+1, 1000000), np.float) simulation_dict['error_log'] = error_log simulation_dict['input_block_size'] = input_block_size simulation_dict['hidden_size'] = hidden_size simulation_dict['learning_rates'] = [] simulation_dict['momenta'] = [] simulation_dict['taus'] = [] simulation_dict['predicted_arrays'] = [] simulation_dict['predicted_arrays_t2'] = [] simulation_dict['predicted_readout_arrays'] = [] simulation_dict['readout_arrays'] = [] simulation_dict['state_arrays'] = [] simulation_dict['delta_arrays'] = [] input_array = SharedArray.SharedNumpyArray((input_block_size*blocks_per_dim[0], input_block_size*blocks_per_dim[0], 3), np.uint8) simulation_dict['input_array'] = input_array input_array_float = SharedArray.SharedNumpyArray((input_block_size*blocks_per_dim[0], input_block_size * blocks_per_dim[0], 3), np.float) simulation_dict['input_array_float'] = input_array_float for (i, bpd) in enumerate(blocks_per_dim): if readout_layer[i]: readout_array_float00 = SharedArray.SharedNumpyArray((bpd*readout_block_size[i], bpd*readout_block_size[i], readout_depth), np.float) simulation_dict['readout_arrays'].append(readout_array_float00) predicted_readout_array_float00 = SharedArray.SharedNumpyArray((bpd*readout_block_size[i], bpd*readout_block_size[i], readout_depth), np.float) simulation_dict['predicted_readout_arrays'].append(predicted_readout_array_float00) # input array 0 is a special case, 3 dimensions because of color input predicted_array0 = SharedArray.SharedNumpyArray((input_block_size*blocks_per_dim[0], input_block_size*blocks_per_dim[0], 3), np.float) simulation_dict['predicted_arrays'].append(predicted_array0) predicted_array2 = SharedArray.SharedNumpyArray((input_block_size*blocks_per_dim[0], input_block_size*blocks_per_dim[0], 3), np.float) simulation_dict['predicted_arrays_t2'].append(predicted_array2) delta_array0 = SharedArray.SharedNumpyArray((input_block_size*blocks_per_dim[0], input_block_size*blocks_per_dim[0], 3), np.float) simulation_dict['delta_arrays'].append(delta_array0) # All the rest is generic for (i, bpd) in enumerate(blocks_per_dim[:-1]): predicted_array1 = SharedArray.SharedNumpyArray((hidden_size*bpd, hidden_size*bpd), np.float) simulation_dict['predicted_arrays'].append(predicted_array1) predicted_array2 = SharedArray.SharedNumpyArray((hidden_size*bpd, hidden_size*bpd), np.float) simulation_dict['predicted_arrays_t2'].append(predicted_array2) delta_array1 = SharedArray.SharedNumpyArray((hidden_size*bpd, hidden_size*bpd), np.float) simulation_dict['delta_arrays'].append(delta_array1) for (i, bpd) in enumerate(blocks_per_dim): state_array0 = SharedArray.SharedNumpyArray((hidden_size*bpd, hidden_size*bpd), np.float) simulation_dict['state_arrays'].append(state_array0) # Base learning rate for (i, bpd) in enumerate(blocks_per_dim): learning_rate = SharedArray.SharedNumpyArray((1, ), np.float) learning_rate[0] = 0.0 simulation_dict['learning_rates'].append(learning_rate) additional_learning_rate = SharedArray.SharedNumpyArray((1, ), np.float) additional_learning_rate[0] = 0.0 simulation_dict['readout_learning_rate'] = additional_learning_rate # Momentum is the same everywhere. momentum = SharedArray.SharedNumpyArray((1, ), np.float) momentum[0] = float(options["momentum"]) simulation_dict['momentum'] = momentum # Tau is the integration constant for the signal integral tau = SharedArray.SharedNumpyArray((1, ), np.float) tau[0] = float(options['tau']) simulation_dict['tau'] = tau context_factor_lateral = SharedArray.SharedNumpyArray((1, ), np.float) context_factor_lateral[0] = 0.0 simulation_dict['context_factor_lateral'] = context_factor_lateral context_factor_feedback = SharedArray.SharedNumpyArray((1, ), np.float) context_factor_feedback[0] = 0.0 simulation_dict['context_factor_feedback'] = context_factor_feedback base_index = [0] for bpd in blocks_per_dim: base_index.append(base_index[-1] + bpd*bpd) # Layer 0 is specific and has to be constructed separately for i in xrange(blocks_per_dim[0] * blocks_per_dim[0]): unit_parameters = create_basic_unit_v1(simulation_dict['learning_rates'][0], momentum, tau, additional_learning_rate) x = (i / blocks_per_dim[0])*input_block_size y = (i % blocks_per_dim[0])*input_block_size dx = input_block_size dy = input_block_size input_block = SharedArray.DynamicView(input_array_float)[x:x+dx, y:y+dy] predicted_block = SharedArray.DynamicView(simulation_dict['predicted_arrays'][0])[x:x+dx, y:y+dy] predicted_block_2 = SharedArray.DynamicView(simulation_dict['predicted_arrays_t2'][0])[x:x+dx, y:y+dy] delta_block = SharedArray.DynamicView(simulation_dict['delta_arrays'][0])[x:x+dx, y:y+dy] if not (predicted_block.shape == (dx, dy, 3)): print predicted_block.shape raise Exception("Block sizes don't agree") k = (i / blocks_per_dim[0])*hidden_size l = (i % blocks_per_dim[0])*hidden_size output_block = SharedArray.DynamicView(simulation_dict['state_arrays'][0])[k:k+hidden_size, l:l+hidden_size] unit_parameters['signal_blocks'].append((input_block, delta_block, predicted_block, predicted_block_2 )) unit_parameters['output_block'] = output_block if readout_layer[0]: # Motor heatmap prediction layer = 0 bpd = blocks_per_dim[layer] readout_teaching_block = SharedArray.DynamicView(simulation_dict['readout_arrays'][layer])[(i / bpd)*readout_block_size[0]:(i / bpd+1)*readout_block_size[0], (i % bpd)*readout_block_size[0]:(i % bpd+1)*readout_block_size[0]] readout_delta_block = SharedArray.SharedNumpyArray_like(readout_teaching_block) predicted_readout_block = SharedArray.DynamicView(simulation_dict['predicted_readout_arrays'][layer])[(i / bpd)*readout_block_size[0]:(i / bpd+1)*readout_block_size[0], (i % bpd)*readout_block_size[0]:(i % bpd+1)*readout_block_size[0]] unit_parameters['readout_blocks'] = [(readout_teaching_block, readout_delta_block, predicted_readout_block)] unit_parameters["layer"] = 0 # End motor heatmap prediction simulation_dict['stage0'].append(unit_parameters) # Layer 0 surround gather_surround(simulation_dict, (base_index[0], blocks_per_dim[0]), radius=lateral_radius, context_factor=context_factor_lateral, exclude_self=exclude_self) # The following layers are more generic for layer in range(1, layers): for i in xrange(blocks_per_dim[layer] * blocks_per_dim[layer]): unit_parameters = create_basic_unit_v1(simulation_dict['learning_rates'][layer], momentum, tau, additional_learning_rate) k = (i / blocks_per_dim[layer])*hidden_size l = (i % blocks_per_dim[layer])*hidden_size output_block = SharedArray.DynamicView(simulation_dict['state_arrays'][layer])[k:k+hidden_size, l:l+hidden_size] unit_parameters['output_block'] = output_block if readout_layer[layer]: # Motor heatmap prediction bpd = blocks_per_dim[layer] readout_teaching_block = SharedArray.DynamicView(simulation_dict['readout_arrays'][layer])[(i / bpd)*readout_block_size[layer]:(i / bpd+1)*readout_block_size[layer], (i % bpd)*readout_block_size[layer]:(i % bpd+1)*readout_block_size[layer]] readout_delta_block = SharedArray.SharedNumpyArray_like(readout_teaching_block) predicted_readout_block = SharedArray.DynamicView(simulation_dict['predicted_readout_arrays'][layer])[(i / bpd)*readout_block_size[layer]:(i / bpd+1)*readout_block_size[layer], (i % bpd)*readout_block_size[layer]:(i % bpd+1)*readout_block_size[layer]] unit_parameters['readout_blocks'] = [(readout_teaching_block, readout_delta_block, predicted_readout_block)] unit_parameters["layer"] = layer # End motor heatmap prediction simulation_dict['stage0'].append(unit_parameters) # Connect to the previous layer connect_forward_and_back_v1(simulation_dict, (base_index[layer-1], blocks_per_dim[layer-1], simulation_dict['predicted_arrays'][layer], simulation_dict['predicted_arrays_t2'][layer]), (base_index[layer], blocks_per_dim[layer]), square_size=fan_in_square_size, radius=fan_in_radius, context_factor=context_factor_feedback) # Layer surround gather_surround(simulation_dict, (base_index[layer], blocks_per_dim[layer]), radius=lateral_radius, context_factor=context_factor_lateral, exclude_self=exclude_self) if send_context_two_layers_back and layer > 1: connect_back(simulation_dict, (base_index[layer], blocks_per_dim[layer]), (base_index[layer-2], blocks_per_dim[layer-2]), square_size=2*fan_in_square_size, radius=2*fan_in_radius, context_factor=context_factor_feedback) # Add the global feedback from the top layer if last_layer_context_to_all: logging.info("Connecting last layer back to everyone") for to_idx in xrange(base_index[layers-1]): for from_idx in range(base_index[layers-1], len(simulation_dict["stage0"])): context_block = simulation_dict['stage0'][from_idx]['output_block'] delta_block2 = SharedArray.SharedNumpyArray_like(context_block) simulation_dict['stage0'][from_idx]['delta_blocks'].append(delta_block2) # Connect the context block to the source simulation_dict['stage0'][to_idx]['context_blocks'].append((context_block, delta_block2, context_factor_feedback)) simulation_dict['stage0_size'] = len(simulation_dict['stage0']) for i in range(simulation_dict['stage0_size']): simulation_dict['stage0'][i]['flags'] = simulation_dict['flags'] unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i], options=options) return simulation_dict def upgrade_readout(simulation_dict): """ Upgrade the dictionary with a parameters for a three layer perceptron for readout :param simulation_dict: :return: """ logging.info("Upgrading readout network to a full perceptron") import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP simulation_dict['stage0_size'] = len(simulation_dict['stage0']) needed = True for i in range(simulation_dict['stage0_size']): if len(simulation_dict['stage0'][i]["MLP_parameters_additional"]['layers']) == 2 and len(simulation_dict['stage0'][i]["MLP_parameters_additional"]['weights']) == 1: nhidden = simulation_dict['stage0'][i]["MLP_parameters_additional"]['layers'][0]['activation'].shape[0]-1 nadditional = simulation_dict['stage0'][i]["MLP_parameters_additional"]['layers'][-1]['activation'].shape[0]-1 layer = {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) } simulation_dict['stage0'][i]["MLP_parameters_additional"]['layers'].insert(1, layer) simulation_dict['stage0'][i]["MLP_parameters_additional"]['weights'] = \ [MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, nhidden), np.float)), MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, nadditional), np.float)), ] else: needed = False if needed: logging.info("Upgrade complete") else: logging.info("Upgrade was not nescessary") def upgrade(simulation_dict): old_log = simulation_dict['error_log'] if old_log.shape[1] < 1000000: new_log = SharedArray.SharedNumpyArray((old_log.shape[0], 1000000), np.float) new_log[:, 0:old_log.shape[1]] = old_log simulation_dict['error_log'] = new_log if "flags" not in simulation_dict.keys(): simulation_dict['flags'] = SharedArray.SharedNumpyArray((16,), np.uint8) for i in range(16): simulation_dict['flags'][i] = 0 if "record_filename" not in simulation_dict.keys(): simulation_dict['record_filename'] = SharedArray.SharedNumpyArray((256,), np.uint8) def apply_options(simulation_dict, options): if "bias_free" in options and options["bias_free"] == "1": for i in range(simulation_dict['stage0_size']): simulation_dict['stage0'][i]["Primary_Predictor_params"]['no_bias'] = True if "Residual_Predictor_params" in simulation_dict['stage0'][i].keys(): simulation_dict['stage0'][i]["Residual_Predictor_params"]['no_bias'] = True if "bias_free_additional" in options and options["bias_free_additional"] == "1": for i in range(simulation_dict['stage0_size']): simulation_dict['stage0'][i]["Readout_Predictor_params"]['no_bias'] = True simulation_dict['options'] = options def upgrade_dictionary_to_ver1_0(simulation_dict): upgrade(simulation_dict) if "version_major" not in simulation_dict.keys() or simulation_dict["version_major"] < 1: logging.info("Simulation dictionary of the old type, automatically upgrading to ver 1.0") if 'learning_rates' not in simulation_dict.keys(): simulation_dict['learning_rates'] = [] if 'momenta' not in simulation_dict.keys(): simulation_dict['momenta'] = [] if 'taus' not in simulation_dict.keys(): simulation_dict['taus'] = [] if 'predicted_arrays' not in simulation_dict.keys(): simulation_dict['predicted_arrays'] = [] if 'predicted_arrays_t2' not in simulation_dict.keys(): simulation_dict['predicted_arrays_t2'] = [] if 'predicted_readout_arrays' not in simulation_dict.keys(): simulation_dict['predicted_readout_arrays'] = [] if 'readout_arrays' not in simulation_dict.keys(): simulation_dict['readout_arrays'] = [] if 'predicted_arrays' not in simulation_dict.keys(): simulation_dict['predicted_arrays'] = [] if 'state_arrays' not in simulation_dict.keys(): simulation_dict['state_arrays'] = [] if 'delta_arrays' not in simulation_dict.keys(): simulation_dict['delta_arrays'] = [] for i in range(PVM_MAX_LAYERS): # max number of layers if "delta_array%02d" % i in simulation_dict.keys(): simulation_dict['delta_arrays'].append(simulation_dict['delta_array%02d' % i]) del simulation_dict['delta_array%02d' % i] if "learning_rate%02d" % i in simulation_dict.keys(): simulation_dict['learning_rates'].append(simulation_dict['learning_rate%02d' % i]) del simulation_dict['learning_rate%02d' % i] if "state_array%02d" % i in simulation_dict.keys(): simulation_dict['state_arrays'].append(simulation_dict['state_array%02d' % i]) del simulation_dict['state_array%02d' % i] if "predicted_readout_array_float%02d" % i in simulation_dict.keys(): simulation_dict['predicted_readout_arrays'].append(simulation_dict['predicted_readout_array_float%02d' % i]) del simulation_dict['predicted_readout_array_float%02d' % i] if "readout_array_float%02d" % i in simulation_dict.keys(): simulation_dict['readout_arrays'].append(simulation_dict['readout_array_float%02d' % i]) del simulation_dict['readout_array_float%02d' % i] if "predicted_array%02d" % i in simulation_dict.keys(): simulation_dict['predicted_arrays'].append(simulation_dict['predicted_array%02d' % i]) simulation_dict['predicted_arrays_t2'].append(SharedArray.SharedNumpyArray_like(simulation_dict['predicted_array%02d' % i])) del simulation_dict['predicted_array%02d' % i] if "motor_delta_array_float" in simulation_dict.keys(): del simulation_dict["motor_delta_array_float"] if "readout_array_float" in simulation_dict.keys(): del simulation_dict["readout_array_float"] if "readout_array_float" in simulation_dict.keys(): del simulation_dict["readout_array_float"] if "predicted_motor_derivative_array_float" in simulation_dict.keys(): del simulation_dict["predicted_motor_derivative_array_float"] if "predicted_readout_array_float" in simulation_dict.keys(): del simulation_dict["predicted_readout_array_float"] if "additional_learning_rate" in simulation_dict.keys(): simulation_dict["readout_learning_rate"] = simulation_dict["additional_learning_rate"] del simulation_dict["additional_learning_rate"] elif "readout_learning_rate" not in simulation_dict.keys(): simulation_dict["readout_learning_rate"] = SharedArray.SharedNumpyArray((1, ), np.float) simulation_dict["readout_learning_rate"][:] = 0.00001 simulation_dict['execution_unit_module'] += "_v1" if not simulation_dict['execution_unit_module'].startswith("PVM_models"): simulation_dict['execution_unit_module'] = "PVM_models."+simulation_dict['execution_unit_module'] ex_unit = importlib.import_module(simulation_dict['execution_unit_module']) for s in range(simulation_dict['stages']): stage = simulation_dict['stage%d' % s] for unit in stage: signal_blocks = unit['signal_blocks'] unit['signal_blocks'] = [] for block in signal_blocks: # Each block: signal_block, delta_block, prediction_t+1, prediction_t+2 unit['signal_blocks'].append([block[0], block[1], block[2], SharedArray.SharedNumpyArray_like(block[2])]) context_blocks = unit['context_blocks'] unit['context_blocks'] = [] for block in context_blocks: # Each block: context_block, delta_block, switching_factor unit['context_blocks'].append([block[0], block[1], block[2]]) readout_blocks = unit['predicted_blocks'] del unit['predicted_blocks'] unit['readout_blocks'] = [] for block in readout_blocks: # Each block: teaching_signal, delta_block, readout_block unit['readout_blocks'].append([block[0], block[1], block[2]]) # Delta blocks can remain unchanged if "learning_rate" in unit.keys(): unit['primary_learning_rate'] = unit.pop("learning_rate") if "momentum" in unit.keys(): unit['primary_momentum'] = unit.pop("momentum") unit['readout_momentum'] = unit['primary_momentum'] if "additional_learning_rate" in unit.keys(): unit['readout_learning_rate'] = unit.pop("additional_learning_rate") else: unit['readout_learning_rate'] = simulation_dict["readout_learning_rate"] # Output block may remain unchanged if "MLP_parameters" in unit.keys(): unit['Primary_Predictor_params'] = unit.pop("MLP_parameters") if "MLP_parameters_res" in unit.keys(): unit['Residual_Predictor_params'] = unit.pop("MLP_parameters_res") if "MLP_parameters_additional" in unit.keys(): unit['Readout_Predictor_params'] = unit.pop("MLP_parameters_additional") unit['flags'] = simulation_dict['flags'] ex_unit.ExecutionUnit.upgrade_to_ver_1(unit) simulation_dict['version_major'] = 1 simulation_dict['version_minor'] = 0 # Remove all the old source files simulation_dict['sources'] = {} logging.info("Upgrade succesfull") else: for s in range(simulation_dict['stages']): stage = simulation_dict['stage%d' % s] for unit in stage: unit['flags'] = simulation_dict['flags'] logging.info("Dictionary already ver 1.0 or above, no need to upgrade") if __name__ == "__main__": pass ================================================ FILE: PVM_framework/PVM_SignalProvider.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import logging from PVM_tools.labeled_movie import FrameCollection import cv2 import os import numpy as np class SimpleSignalProvider(AbstractExecutionManager.AbstractSignalProvider): def __init__(self, frame_resolution, heatmap_resolution, files, storage, channel="default", remove_files=True, reverse=False): """ Handy object to provide input frames and supervising heatmaps efficiently :param frame_resolution: :param heatmap_resolution: :param files: :param channel: """ self.files = files self.frame_resolution = frame_resolution self.heatmap_resolution = heatmap_resolution self.channel = channel self.frames = [] self.nframes = 0 self.masks = [] self.index = 0 self.remove_files = remove_files self.reverse = reverse self.storage=storage def start(self): """ Collect and preprocess all the data :return: """ for (labeled_file, channel) in self.files: logging.info("Incorporating file %s" % (labeled_file)) local_labeled_file = self.storage.get(labeled_file) if local_labeled_file is not None: self.fc = FrameCollection() self.fc.load_from_file(local_labeled_file) self.nframes += len(self.fc) for i in xrange(len(self.fc)): img1 = self.fc.Frame(i).get_image(channel=channel) img = cv2.resize(img1, dsize=self.frame_resolution, interpolation=cv2.INTER_CUBIC) label = self.fc.Frame(i).get_label(channel=channel) if label is not None: label.set_image_shape(shape=img1.shape) mask = self.fc.Frame(i).get_label(channel=channel).get_mask() mask = cv2.resize(mask, dsize=self.heatmap_resolution, interpolation=cv2.INTER_CUBIC) else: mask = np.zeros(self.heatmap_resolution, dtype=np.uint8) self.masks.append(mask) self.frames.append(img) if self.remove_files: os.remove(local_labeled_file) def get_signal(self, name, time): """ Return the requested signal. If time is zero, return the current signal, otherwise return past of future signals as indicated by the time parameter :param name: :param time: :return: """ if name == "frame": return self.frames[(self.index+time) % self.nframes] elif name == "mask": return self.masks[(self.index+time) % self.nframes] else: raise Exception("Unknown signal type") def get_length(self): return self.nframes def advance(self): """ Move current index by one step forward :return: """ if self.reverse: self.index = (self.index - 1) % self.nframes else: self.index = (self.index + 1) % self.nframes def finish(self): """ Cleanup :return: """ pass def reset(self): self.index = 0 def get_index(self): """ Return the index of the current frame in the sequence :return: """ return self.index class StereoSignalProvider(AbstractExecutionManager.AbstractSignalProvider): def __init__(self, frame_resolution, heatmap_resolution, files, storage, channel="default", remove_files=True, only_one_channel=""): """ Handy object to provide input frames and supervising heatmaps efficiently :param frame_resolution: :param heatmap_resolution: :param files: :param channel: :return: """ self.files = files self.frame_resolution = frame_resolution self.frame_resolution_half = (frame_resolution[0], frame_resolution[1]/2) self.heatmap_resolution = heatmap_resolution self.channel = channel self.frames = [] self.nframes = 0 self.masks = [] self.index = 0 self.remove_files = remove_files self.only_one_channel = only_one_channel self.storage=storage def start(self): """ Collect and preprocess all the data :return: """ for (labeled_file, channel) in self.files: logging.info("Incorporating file %s" % (labeled_file)) local_labeled_file = self.storage.get(labeled_file) if local_labeled_file is not None: self.fc = FrameCollection() self.fc.load_from_file(local_labeled_file) self.nframes += len(self.fc) for i in xrange(len(self.fc)): img1 = self.fc.Frame(i).get_image(channel="left") imgl = cv2.resize(img1, dsize=self.frame_resolution_half, interpolation=cv2.INTER_CUBIC) img1 = self.fc.Frame(i).get_image(channel="right") imgr = cv2.resize(img1, dsize=self.frame_resolution_half, interpolation=cv2.INTER_CUBIC) if self.only_one_channel == "left": imgr *= 0 if self.only_one_channel == "right": imgl *= 0 img = self.interlace_two_images(imgl, imgr) label = self.fc.Frame(i).get_label(channel="left") if label is not None: label.set_image_shape(shape=img1.shape) mask = label.get_mask() mask = cv2.resize(mask, dsize=self.heatmap_resolution, interpolation=cv2.INTER_CUBIC) else: mask = np.zeros(self.heatmap_resolution, dtype=np.uint8) self.masks.append(mask) self.frames.append(img) if self.remove_files: os.remove(local_labeled_file) def interlace_two_images(self, img1, img2): interlacedimg = np.zeros((img1.shape[0]+img2.shape[0], img1.shape[1], img1.shape[2]), dtype=np.uint8) interlacedimg[0::2, :, :] = img1 interlacedimg[1::2, :, :] = img2 return interlacedimg def get_signal(self, name, time): """ Return the requested signal. If time is zero, return the current signal, otherwise return past of future signals as indicated by the time parameter :param name: :param time: :return: """ if name == "frame": return self.frames[(self.index+time) % self.nframes] elif name == "mask": return self.masks[(self.index+time) % self.nframes] else: raise Exception("Unknown signal type") def get_length(self): return self.nframes def advance(self): """ Move current index by one step forward :return: """ self.index = (self.index + 1) % self.nframes def finish(self): """ Cleanup :return: """ pass def reset(self): self.index = 0 def get_index(self): """ Return the index of the current frame in the sequence :return: """ return self.index class TripleSignalProvider(AbstractExecutionManager.AbstractSignalProvider): def __init__(self, frame_resolution, heatmap_resolution, files1, files2, files3, storage, channel="default", remove_files=True): """ Handy object to provide input frames and supervising heatmaps efficiently :param frame_resolution: :param heatmap_resolution: :param files: :param channel: :return: """ self.files1 = files1 self.files2 = files2 self.files3 = files3 self.frame_resolution = frame_resolution self.heatmap_resolution = heatmap_resolution self.channel = channel self.frames = [] self.nframes = 0 self.masks = [] self.index = 0 self.remove_files = remove_files self.storage = storage def buffer_files(self, idx, channel, files): for (labeled_file, channel) in files: logging.info("Incorporating file %s" % (labeled_file)) local_labeled_file = self.storage.get(labeled_file) if local_labeled_file is not None: self.fc = FrameCollection() self.fc.load_from_file(local_labeled_file) self.nframes += len(self.fc) for i in xrange(len(self.fc)): img1 = self.fc.Frame(i).get_image(channel=channel) img = cv2.resize(img1, dsize=self.frame_resolution, interpolation=cv2.INTER_CUBIC) label = self.fc.Frame(i).get_label(channel=channel) label.set_image_shape(shape=img1.shape) mask = self.fc.Frame(i).get_label(channel=channel).get_mask() mask = cv2.resize(mask, dsize=self.heatmap_resolution, interpolation=cv2.INTER_CUBIC) mask3d = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=mask.dtype) mask3d[:, :, idx] = mask self.masks.append(mask3d) self.frames.append(img) if self.remove_files: os.remove(local_labeled_file) def start(self): """ Collect and preprocess all the data :return: """ self.buffer_files(0, channel=self.channel, files=self.files1) self.buffer_files(1, channel=self.channel, files=self.files2) self.buffer_files(2, channel=self.channel, files=self.files3) def get_signal(self, name, time): """ Return the requested signal. If time is zero, return the current signal, otherwise return past of future signals as indicated by the time parameter :param name: :param time: :return: """ if name == "frame": return self.frames[(self.index+time) % self.nframes] elif name == "mask": return self.masks[(self.index+time) % self.nframes] else: raise Exception("Unknown signal type") def get_length(self): return self.nframes def advance(self): """ Move current index by one step forward :return: """ self.index = (self.index + 1) % self.nframes def finish(self): """ Cleanup :return: """ pass def reset(self): self.index = 0 def get_index(self): """ Return the index of the current frame in the sequence :return: """ return self.index ================================================ FILE: PVM_framework/PVM_Storage.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import ssl import socket import sys import traceback import os import logging import re import random import hashlib import errno import shutil def get_S3_credentials(): """ The function will attempt to load credentials from ~/.aws/credentials (as well as environment variables, boto config files and other places) If succesfull will return a credentials object. :return: """ from boto.provider import Provider from boto.sts.credentials import Credentials provider = Provider('aws') if provider.access_key and provider.secret_key: cred = Credentials() cred.access_key = provider.access_key cred.secret_key = provider.secret_key return cred else: logging.getLogger(__name__).info("Could not load permanent credentials") return None class Storage(object): _pattern = re.compile("[a-zA-Z0-9._\\-/]*$") def __init__(self, local_storage_folder="~/PVM_data", S3_bucket="technical.services.braincorporation.net"): """ Initialize storage object. All the elements will by stored in the local_storage_folder. If in addition an S3 bucket and credentials are given, the local storage will act as local cache for S3. :param local_storage_folder: A folder where the files are stored/mirrored :param S3_bucket: Optional S3 bucket where the data will be stored :param S3_credentials: Credentials to access the bucket :return: """ if S3_bucket is not None: S3_credentials = get_S3_credentials() if S3_credentials is not None and S3_credentials.access_key and S3_credentials.secret_key: import boto logging.getLogger('boto').setLevel(logging.CRITICAL) self.connector = boto.connect_s3(S3_credentials.access_key, S3_credentials.secret_key, security_token=S3_credentials.session_token, calling_format='boto.s3.connection.OrdinaryCallingFormat') # getting bucket with disabled validation to avoid connection error:'S3ResponseError:403 URLBlocked' self.bucket = self.connector.get_bucket(S3_bucket, validate=False) else: self.bucket = None else: self.bucket = None self.local_storage = os.path.expanduser(local_storage_folder) if not os.path.exists(self.local_storage): os.mkdir(self.local_storage) def is_valid_key(self, key): return self._pattern.match(key) is not None def exists(self, key): """ Do not check existence of a folder because there are no folders in S3 """ assert self.is_valid_key(key) from boto.exception import S3ResponseError try: k = self.bucket.get_key(key) return k is not None except S3ResponseError: # When using a guest account, checking whether an object exists # will result in a 503 Forbidden exception if it does not exist return False def _ensure_containing_folder_exists(self, filepath): """ Make sure that the folder containing filepath exists already to avoid errors when creating the file. """ directory = os.path.dirname(filepath) try: os.makedirs(directory) except OSError as err: if err.errno != errno.EEXIST: raise def get(self, path, to_path=None, force=False): """ Get a file from local/remote storage. The file will get downloaded from S3 if bucket is in use, to a local storage directory. Path to the file in local storage directory will be returned. :param path: :param to_path: :param force: :return: """ logging.info("Storage received a request to get file %s" % path) if self.bucket is None: file_p = os.path.join(self.local_storage, path) logging.info("Loading file %s" % file_p) if not os.path.exists(file_p): # maybe then such file actually exist in the file system? if not os.path.exists(path): logging.error("File %s does not exist!" % path) raise Exception("Requested file %s does not exist" % path) else: return path else: return file_p else: if to_path is None: dest_path = os.path.join(self.local_storage, path) else: dest_path = to_path if os.path.isfile(dest_path): if self.exists(path) and force: os.remove(dest_path) elif force: raise Exception("Forcing download but the original file does not exist on S3") else: return dest_path try: if self.exists(path) and not path.endswith("/"): while True: try: self.download(to_file=dest_path, key=path, progress_callback=self._progress_download) break except ssl.SSLError, socket.error: logging.error("Download timeout, trying again! file %s" % path) continue logging.info("Downloaded file %s to %s" % (path, dest_path)) return dest_path else: if not os.path.exists(path): logging.error("File %s does not exist!" % path) raise Exception("Requested file %s does not exist" % path) else: logging.info("Using file found in the local file system. %s" % path) return path except: traceback.print_exc(file=sys.stdout) return None def put(self, path=None, from_path=None, to_folder=None, overwrite=False): """ Put a file to local/remot storage. If using an S3 bucket the file will be uploaded. If not it will be copied to local storage :param path: path (key) within the local cache structure (same are remote S3 path) :type path: str :param from_path: (optional) path to the file that should be uploaded :type from path: str :param to_folder: (optional) remote directory to which the file needs to be uploaded :type to_folder: str :param overwrite: if True a file will be overwritten if it already exists remotely :type overwrite: bool :return: True if successful, None if error """ if path == "/": path = "" # Just copy to the right directory if path is None: dst = os.path.join(self.local_storage, to_folder) dst = os.path.join(dst, os.path.basename(from_path)) else: dst = os.path.join(self.local_storage, path) if os.path.isfile(dst) and not overwrite: logging.error("File %s already exists, not overwriting" % dst) return None self._ensure_containing_folder_exists(dst) if not (os.path.isfile(dst) and os.path.samefile(from_path, dst)): shutil.copy2(from_path, dst) if self.bucket is not None: if from_path is None: source_path = os.path.join(self.local_storage, path) else: source_path = from_path if to_folder is None: to_folder = os.path.dirname(path) if not os.path.isfile(source_path): logging.error("File %s does not exists" % source_path) return None while True: try: self.upload(from_file=source_path, to_folder=to_folder, progress_callback=self._progress_upload, overwrite=overwrite) break except ssl.SSLError, socket.error: logging.error("Upload timeout, trying again!, file %s" % source_path) continue logging.info("Uploaded file %s to folder %s" % (source_path, to_folder)) return True def download(self, to_file, key, progress_callback=None): """ Download a file from S3 to a local storage :param to_file: filename in which to store the downloaded file (str) :param key: the key to download from the bucket (str) :param progress_callback: callback function printing progress :return: """ from boto.s3.key import Key k = Key(self.bucket, key) self._ensure_containing_folder_exists(to_file) tmpfile = to_file + '.tmp.%s' % hashlib.sha1(str(random.getrandbits(256))).hexdigest() try: k.get_contents_to_filename(filename=tmpfile, cb=progress_callback, num_cb=100) except: if os.path.exists(tmpfile): os.remove(tmpfile) raise # download is successful, move the temporary file to desired location os.rename(tmpfile, to_file) def upload(self, from_file, to_folder, overwrite=False, progress_callback=None): """ Upload a file to S3 bucket. S3 doen't have concept of folder so /path/name.txt is just a long filename. It means that 'folders' will be created automatically if they do not exists :param from_file: path to the file being uploaded :param to_folder: S3 "folder" to which the file should go. :param overwrite: overwrite if True :param progress_callback: callback function printing progress :return: """ from boto.s3.key import Key custom_key_name = os.path.basename(from_file) keyname = os.path.join(to_folder, custom_key_name) assert self.is_valid_key(keyname) # check that file exists if not overwrite and self.exists(keyname): raise Exception('Can not upload, file %s already exists on amazon' % keyname) logging.getLogger(__name__).info('UI:Uploading to "%s"...' % keyname) k = Key(self.bucket, keyname) k.set_contents_from_filename(from_file, cb=progress_callback, num_cb=100) return True def _progress_download(self, elapsed, total): """ Prints the download progress :param elapsed: :param total: :return: """ print "Downloading %d %% \r" % (elapsed*100/max(total, 1)), sys.stdout.flush() def _progress_upload(self, elapsed, total): """ Prints the upload progress :param elapsed: :param total: :return: """ print "Uploading %d %% \r" % (elapsed*100/max(total, 1)), sys.stdout.flush() def list_path(self, path): """ List all the keys in the remote path :param path: path (key) in the local or remote storage :type path: str :return: list of keys :rtype: list """ if path == "/": path = "" result = [] if self.bucket is None: for root, dirs, files in os.walk(self.local_storage, topdown=False): for name in files: result.append(os.path.join(root, name)) else: assert self.is_valid_key(path) keys_list = self.bucket.list(prefix=path) result = [key.name for key in keys_list if os.path.normpath(key.name) != os.path.normpath(path)] return result def remove(self, key, check_existence=True): """ Remove a file from local/remote storage :param key: :param check_existence: """ if self.bucket is None: if check_existence and not os.path.exists(os.path.join(self.local_storage, key)): raise Exception('Can not remove non-existent key %s' % key) os.remove(os.path.join(self.local_storage, key)) else: from boto.s3.key import Key if check_existence and not self.exists(key): raise Exception('Can not remove non-existent key %s' % key) k = Key(self.bucket) k.key = key self.bucket.delete_key(k) def test_storage_local(): """ """ S = Storage(local_storage_folder="~/PVM_test") contents = S.list_path("") assert contents == [] S.put(path="", from_path=os.path.abspath(__file__)) contents = S.list_path("") assert len(contents) == 1 S.remove(__file__) contents = S.list_path("") assert len(contents) == 0 def test_storage_remote(): """ """ S = Storage(local_storage_folder="~/PVM_test_local_remote", S3_bucket="technical.services.braincorporation.net", S3_credentials=get_S3_credentials()) contents = S.list_path("") N = len(contents) S.put(path="", from_path=os.path.abspath(__file__), overwrite=True) contents = S.list_path("") assert len(contents) == (N + 1) S.remove(__file__) contents = S.list_path("") assert len(contents) == N if __name__ == "__main__": test_storage_local() test_storage_remote() ================================================ FILE: PVM_framework/PVM_datasets.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.PVM_Storage as PVM_Storage import logging sets = { "face_training": [["PVM_data/face01.pkl", "default"], ["PVM_data/face03.pkl", "default"], ["PVM_data/no_target.pkl", "default"], ["PVM_data/face16.pkl", "default"], ["PVM_data/face17.pkl", "default"], ["PVM_data/face18.pkl", "default"], ["PVM_data/face19.pkl", "default"], ["PVM_data/no_target_01.pkl", "default"], ["PVM_data/face20.pkl", "default"], ["PVM_data/face21.pkl", "default"], ], "face_testing": [["PVM_data/face02.pkl", "default"], ["PVM_data/face04.pkl", "default"], ["PVM_data/face05.pkl", "default"], ["PVM_data/face06.pkl", "default"], ["PVM_data/face07.pkl", "default"], ["PVM_data/face08.pkl", "default"], ["PVM_data/face09.pkl", "default"], ["PVM_data/face10.pkl", "default"], ["PVM_data/face11.pkl", "default"], ["PVM_data/face12.pkl", "default"], ["PVM_data/face13.pkl", "default"], ["PVM_data/face14.pkl", "default"], ["PVM_data/face15.pkl", "default"], ["PVM_data/face22.pkl", "default"], ["PVM_data/face23.pkl", "default"], ["PVM_data/face24.pkl", "default"], ], "face_ex_testing": [["PVM_data/face25.pkl", "default"], ["PVM_data/face26.pkl", "default"], ["PVM_data/face27.pkl", "default"], ["PVM_data/face28.pkl", "default"], ["PVM_data/face29.pkl", "default"], ["PVM_data/face30.pkl", "default"], ["PVM_data/face31.pkl", "default"], ["PVM_data/face32.pkl", "default"], ["PVM_data/face33.pkl", "default"], ], "face_additional": [], "green_ball_training": [["PVM_data/green_ball_long.pkl", "default"], ["PVM_data/green_ball_on_grass.pkl", "default"], ["PVM_data/green_ball_test_14.pkl", "default"], ["PVM_data/green_ball_test_15.pkl", "default"], ["PVM_data/green_ball_test_16.pkl", "default"], ], "green_ball_testing": [["PVM_data/green_ball_test.pkl", "default"], ["PVM_data/green_ball_test_01.pkl", "default"], ["PVM_data/green_ball_test_02.pkl", "default"], ["PVM_data/green_ball_test_03.pkl", "default"], ["PVM_data/green_ball_test_04.pkl", "default"], ["PVM_data/green_ball_test_05.pkl", "default"], ["PVM_data/green_ball_test_06.pkl", "default"], ["PVM_data/green_ball_test_07.pkl", "default"], ["PVM_data/green_ball_test_08.pkl", "default"], ["PVM_data/green_ball_test_09.pkl", "default"], ["PVM_data/green_ball_test_10.pkl", "default"], ["PVM_data/green_ball_test_11.pkl", "default"], ["PVM_data/green_ball_test_12.pkl", "default"], ["PVM_data/green_ball_test_13.pkl", "default"], ["PVM_data/green_ball_01_small.pkl", "default"], ["PVM_data/green_ball_bc_office.pkl", "default"], ], "green_ball_ex_testing": [["PVM_data/green_ball_test_17.pkl", "default"], ["PVM_data/green_ball_test_18.pkl", "default"], ["PVM_data/green_ball_test_19.pkl", "default"], ["PVM_data/green_ball_test_20.pkl", "default"], ["PVM_data/green_ball_test_21.pkl", "default"], ["PVM_data/green_ball_test_22.pkl", "default"], ["PVM_data/green_ball_test_23.pkl", "default"], ["PVM_data/green_ball_test_24.pkl", "default"], ["PVM_data/green_ball_test_25.pkl", "default"], ["PVM_data/green_ball_test_26.pkl", "default"], ["PVM_data/green_ball_test_27.pkl", "default"], ["PVM_data/green_ball_test_28.pkl", "default"], ["PVM_data/green_ball_test_29.pkl", "default"], ], "green_ball_additional": [["PVM_data/blue_ball_on_grass_daytime.pkl", "default"], ["PVM_data/blue_ball_at_home_02.pkl", "default"], ["PVM_data/blue_ball_at_home_01.pkl", "default"], ], "stop_sign_training": [["PVM_data/stop01.pkl", "default"], ["PVM_data/stop03.pkl", "default"], ["PVM_data/stop05.pkl", "default"], ["PVM_data/stop07.pkl", "default"], ["PVM_data/stop09.pkl", "default"], ["PVM_data/stop11.pkl", "default"], ["PVM_data/stop13.pkl", "default"], ["PVM_data/stop15.pkl", "default"], ["PVM_data/stop17.pkl", "default"], ["PVM_data/stop19.pkl", "default"], ["PVM_data/stop21.pkl", "default"], ["PVM_data/stop23.pkl", "default"], ["PVM_data/stop25.pkl", "default"], ["PVM_data/stop27.pkl", "default"], ["PVM_data/stop29.pkl", "default"], ["PVM_data/stop32.pkl", "default"], ["PVM_data/stop34.pkl", "default"], ["PVM_data/stop36.pkl", "default"], ["PVM_data/stop38.pkl", "default"], ["PVM_data/stop40.pkl", "default"], ], "stop_sign_testing": [["PVM_data/stop02.pkl", "default"], ["PVM_data/stop04.pkl", "default"], ["PVM_data/stop06.pkl", "default"], ["PVM_data/stop08.pkl", "default"], ["PVM_data/stop10.pkl", "default"], ["PVM_data/stop12.pkl", "default"], ["PVM_data/stop14.pkl", "default"], ["PVM_data/stop16.pkl", "default"], ["PVM_data/stop18.pkl", "default"], ["PVM_data/stop20.pkl", "default"], ["PVM_data/stop22.pkl", "default"], ["PVM_data/stop24.pkl", "default"], ["PVM_data/stop26.pkl", "default"], ["PVM_data/stop28.pkl", "default"], ["PVM_data/stop30.pkl", "default"], ["PVM_data/stop33.pkl", "default"], ["PVM_data/stop35.pkl", "default"], ["PVM_data/stop37.pkl", "default"], ["PVM_data/stop39.pkl", "default"], ], "stop_sign_additional": [], "stop_sign_ex_testing": [["PVM_data/stop41.pkl", "default"], ["PVM_data/stop42.pkl", "default"], ["PVM_data/stop43.pkl", "default"], ["PVM_data/stop44.pkl", "default"], ["PVM_data/stop45.pkl", "default"], ["PVM_data/stop46.pkl", "default"], ["PVM_data/stop47.pkl", "default"], ["PVM_data/stop48.pkl", "default"], ["PVM_data/stop49.pkl", "default"], ["PVM_data/stop50.pkl", "default"], ["PVM_data/stop51.pkl", "default"], ], "short_training": [["PVM_data/stop40.pkl", "default"]], "short_testing": [["PVM_data/stop41.pkl", "default"]], "short_additional": [], "non_spec_training": [["PVM_data/no_target.pkl", "default"], ["PVM_data/no_target_01.pkl", "default"], ["PVM_data/green_ball_long.pkl", "default"], ["PVM_data/stop01.pkl", "default"], ["PVM_data/green_ball_on_grass.pkl", "default"], ["PVM_data/stop03.pkl", "default"], ["PVM_data/face01.pkl", "default"], ["PVM_data/stop05.pkl", "default"], ["PVM_data/face03.pkl", "default"], ["PVM_data/stop07.pkl", "default"], ["PVM_data/face16.pkl", "default"], ["PVM_data/stop09.pkl", "default"], ["PVM_data/face17.pkl", "default"], ["PVM_data/stop11.pkl", "default"], ["PVM_data/face18.pkl", "default"], ["PVM_data/stop13.pkl", "default"], ["PVM_data/green_ball_test_14.pkl", "default"], ["PVM_data/stop15.pkl", "default"], ["PVM_data/green_ball_test_15.pkl", "default"], ["PVM_data/stop17.pkl", "default"], ["PVM_data/green_ball_test_16.pkl", "default"], ["PVM_data/stop19.pkl", "default"], ["PVM_data/stop21.pkl", "default"], ["PVM_data/stop23.pkl", "default"], ], "non_spec_testing": [["PVM_data/stop41.pkl", "default"]], "non_spec_additional": [], } sets["stop_sign_full_testing"] = sets["stop_sign_testing"] + sets["stop_sign_ex_testing"] sets["green_ball_full_testing"] = sets["green_ball_testing"] + sets["green_ball_ex_testing"] sets["face_full_testing"] = sets["face_testing"] + sets["face_ex_testing"] sets["stop_sign_fast_testing"] = [sets["stop_sign_testing"][0]] sets["green_ball_fast_testing"] = [sets["green_ball_testing"][0]] sets["face_fast_testing"] = [sets["face_testing"][0]] class PVMDataset(object): def __init__(self, name, storage=None): logging.info("Parsing dataset info on %s" % name) self.name = name self.all = [] if name+"_training" in sets.keys(): # ok so we have a good dataset name self.training = sets[name+"_training"] self.testing = sets[name+"_testing"] self.additional = sets[name+"_additional"] logging.info("Dataset appears to be defined withing datasets") elif name == "all": self.training = [] self.testing = [] self.additional = [] for ds in ("stop_sign", "green_ball", "face"): self.training.extend(sets["%s_training" % ds]) self.testing.extend(sets["%s_testing" % ds]) self.testing.extend(sets["%s_ex_testing" % ds]) self.additional.extend(sets["%s_additional" % ds]) logging.info("Dataset is a special keyword") elif name == "all_short": self.training = [] self.testing = [] self.additional = [] for ds in ("stop_sign", "green_ball", "face"): self.training.append(sets["%s_training" % ds][0]) self.testing.append(sets["%s_testing" % ds][0]) self.additinal.append(sets["%s_testing" % ds][0]) logging.info("Dataset is a special keyword") else: # it does not match anything but maybe it is just a filename directly logging.info("Dataset is unknown, trying a file") try: logging.info("Trying to download the file %s" % name) filename = storage.get(name) logging.info("got file %s" % filename) except: logging.error("Invalid dataset") raise(Exception("Invalid dataset")) if filename is not None: # the file is valid logging.info("Got the file") self.training = [[name, "default"]] self.testing = [[name, "default"]] self.additional = [[name, "default"]] else: logging.error("Invalid dataset") raise(Exception("Invalid dataset")) self.all.extend(self.training) self.all.extend(self.testing) self.all.extend(self.additional) if __name__ == "__main__": import PVM_tools.labeled_movie as lm TS = PVM_Storage.Storage() for set in sets.keys(): print "Set :" + str(set) set_l = 0 for element in sets[set]: local_path = TS.get(element[0]) fc = lm.FrameCollection() fc.load_from_file(local_path) fc.write_to_file(local_path) print "File %s, %d frames %f minutes" % (local_path, len(fc), len(fc)/(25.0*60)) set_l += len(fc) print "Set %s, %f frames, %f minutes" % (set, set_l, set_l/(25.0*60)) ================================================ FILE: PVM_framework/PVM_debug_console.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.SharedArray as SharedArray import PVM_framework.PVM_Create as PVM_Create import os import sys import argparse import pprint import cmd import traceback import time import logging class InteractiveDictionaryExplorer(cmd.Cmd): use_rawinput = False def __init__(self, stdin=None, stdout=None, infilename=None, dict=None, pprint=False, sprint=False, nans=None, gt=None, lt=None, abss=None, filter_name=None, upgrade=False): """ Console object used to traverse the simulation dictionary. Can be used standalone on a saved distinary or live on a running simulation by logging into the debug port. :param stdin: :param stdout: :param infilename: :param dict: :param pprint: :param sprint: :param nans: :param gt: :param lt: :param abss: :param filter_name: :return: """ if stdin is not None: self.stdin = stdin else: self.stdin=sys.stdin self.use_rawinput=True if stdout is not None: self.stdout = stdout else: self.stdout=sys.stdout cmd.Cmd.__init__(self, stdin=self.stdin, stdout=self.stdout) if dict is None and infilename is None: sys.stderr.write("dict and infilename cannot be empty at the same time") if dict is not None: self.dict = dict self.filename = str(infilename) elif os.path.exists(infilename) and os.path.isfile(infilename): # This import below needs to go here as otherwise it would # lead to a circular import and failure. import PVM_framework.CoreUtils as CoreUtils self.dict = CoreUtils.load_model(infilename) if upgrade: import PVM_framework.PVM_Create as PVM_Create PVM_Create.upgrade_dictionary_to_ver1_0(self.dict) else: sys.stderr.write("Input file not found\n") self.pprint = pprint self.sprint = sprint self.filename = str(infilename) self.gt = None self.lt = None self.nans = None self.filter_name = None self.abss=None if gt is not None: self.gt = float(gt) if lt is not None: self.lt = float(lt) if nans is not None: self.nans = True if filter_name is not None: self.filter_name = filter_name if abss is not None: self.abss = float(abss) self.text_bold = '\033[1m' self.text_end = '\033[0m' self.prompt = self.text_bold + str(self.dict['name'])+"/$ " + self.text_end self.current_element = self.dict self.pwd = "/" logging.info("Created an interactive dictionary explorer session") def getsizeof(self, element): size=0 try: size = element.nbytes except: size = sys.getsizeof(element) return size def getshapeof(self, element): shape = "---" try: shape = str(element.shape) except: shape = "---" return shape def gettypename(self, element): name = "" try: name = element.__class__.__name__ except: name = type(element).__name__ return name def emptyline(self): pass def do_ls(self, line, quiet=False): """ List the contents of the current element in the dictionary """ if line.startswith("/"): listed_element = self.getsubelement(self.dict, line) else: listed_element = self.getsubelement(self.current_element, line) if not quiet: print >>self.stdout, "%-*s " % (30, "Element name"), print >>self.stdout, "%*s " % (20, "type"), print >>self.stdout, "%*s " % (12, "shape"), print >>self.stdout, "%*s " % (12, "size (bytes)"), print >>self.stdout, "%*s " % (15, "id") print >>self.stdout, "%-*s " % (30, "."), print >>self.stdout, "%*s " % (20, self.gettypename(listed_element)), print >>self.stdout, "%*s " % (12, self.getshapeof(listed_element)), print >>self.stdout, "%*s " % (12, self.getsizeof(listed_element)), print >>self.stdout, "%*s " % (15, id(listed_element)) if isinstance(listed_element, dict): for key in listed_element.keys(): element = listed_element[key] print >>self.stdout, "%-*s " % (30, key), print >>self.stdout, "%*s " % (20, self.gettypename(element)), print >>self.stdout, "%*s " % (12, self.getshapeof(element)), print >>self.stdout, "%*s " % (12, self.getsizeof(element)), print >>self.stdout, "%*s " % (15, id(element)) elif isinstance(listed_element, list): for i, element in enumerate(listed_element): print >>self.stdout, "%-*s " % (30, str(i)), print >>self.stdout, "%*s " % (20, self.gettypename(element)), print >>self.stdout, "%*s " % (12, self.getsizeof(element)), print >>self.stdout, "%*s " % (12, self.getshapeof(element)), print >>self.stdout, "%*s " % (15, id(element)) elif isinstance(listed_element, tuple): for i, element in enumerate(listed_element): print >>self.stdout, "%-*s " % (30, str(i)), print >>self.stdout, "%*s " % (20, self.gettypename(element)), print >>self.stdout, "%*s " % (12, self.getsizeof(element)), print >>self.stdout, "%*s " % (12, self.getshapeof(element)), print >>self.stdout, "%*s " % (15, id(element)) else: pprint.pprint(listed_element, stream=self.stdout) def do_cd(self, path): """ Change the current element in the dictionary """ wd = "" current_dir = self.dict if path.endswith("/") and len(path)>1: path = path[:-1] if path.startswith("/"): wd = path elif path == ".." or path == "../": wd = "/"+"/".join(self.pwd.split("/")[:-1]) elif path == "../.." or path == "../../": wd = "/"+"/".join(self.pwd.split("/")[:-2]) else: wd = self.pwd + "/" + path subdirs = wd.split("/") try: for subdir in subdirs: if subdir.isdigit(): i = int(subdir) current_dir = current_dir[i] elif subdir != '': current_dir = current_dir[subdir] self.current_element = current_dir self.pwd = wd.replace("//", "/") self.prompt = self.text_bold + self.dict['name'] + self.pwd + "$ " + self.text_end except: print >>self.stdout, "Invalid path" def getsubelement(self, element, path): current_dir = element subdirs = path.split("/") try: for subdir in subdirs: if subdir.isdigit(): i = int(subdir) current_dir = current_dir[i] elif subdir != '': current_dir = current_dir[subdir] return current_dir except: print >>self.stdout, "Invalid path" def getsubelements(self, element): if isinstance(element, dict): return element.keys() if isinstance(element, list): return map(lambda x: str(x), range(len(element))) return [] def generic_complete(self, text, line, beginidx, endidx): if "/" in line: (cmd, arg) = line.split(" ") if not arg.endswith("/"): arg = "/".join(arg.split("/")[:-1]) element = self.getsubelement(self.current_element, arg) prefix = "/".join(text.split("/")[:-1]) else: element = self.current_element prefix = "" all_completions = map(lambda x: prefix + x, self.getsubelements(element)) if not text: return all_completions else: return [c for c in all_completions if c.startswith(text)] def complete_cd(self, text, line, beginidx, endidx): return self.generic_complete(text, line, beginidx, endidx) def complete_ls(self, text, line, beginidx, endidx): return self.generic_complete(text, line, beginidx, endidx) def do_cat(self, line): try: print >>self.stdout, self.current_element[line] except: print >>self.stdout, "Cound not access the element" def do_debug_create(self, line): """ Creates a debug hookup. Called in a given context needs: Nescessary arguments: filename='...' object=name of the object in the given context interval=number of steps between successive dumps Optional: phase=phase shift of the dump (default 0) E.g.: debug_create filename='my_debug.p' object=weights0 interval=10 phase=5 """ try: params = line.split(" ") args = {} for param in params: (L, R) = param.split('=') args[L]=R if "filename" not in args.keys(): raise if "object" not in args.keys(): raise if "interval" not in args.keys(): raise if 'phase' not in args.keys(): args['phase'] = 0 except: print >>self.stdout, "Error processing arguments" logging.debug("Exception in debug hookup creation, error processing arguments") try: hookup = {} hookup['filename'] = args['filename'] # Will raise an exception unless the call below succedes self.current_element[args['object']].view(np.ndarray) hookup['object'] = self.current_element[args['object']] hookup['interval'] = int(args['interval']) hookup['phase'] = int(args['phase']) hookup['path'] = self.pwd + '/' + args['object'] self.dict['debug_infrastructure']['disabled_hookups'][args['filename']] = hookup except: print >>self.stdout, "Error making a debug hookup" traceback.print_exc(file=self.stdout) logging.debug("Exception in debug hookup creation") logging.info("Created a debug hookup with parameters " + line) def do_debug_disable(self, line): """ Disable a currently enabled debug hookup """ try: hookup = self.dict['debug_infrastructure']['enabled_hookups'][line] self.dict['debug_infrastructure']['disabled_hookups'][line]=hookup del self.dict['debug_infrastructure']['enabled_hookups'][line] print >>self.stdout, "Disabled "+line except: print >>self.stdout, "Error disabling a debug hookup" traceback.print_exc(file=self.stdout) logging.debug("Exception while disabling a debug hookup " + line) logging.info("Disabled a debug hookup " + line) def do_debug_enable(self, line): """ Enable a currently disabled debug hookup """ try: hookup = self.dict['debug_infrastructure']['disabled_hookups'][line] self.dict['debug_infrastructure']['enabled_hookups'][line]=hookup del self.dict['debug_infrastructure']['disabled_hookups'][line] print >>self.stdout, "Enabled "+line except: print >>self.stdout, "Error enabling a debug hookup" traceback.print_exc(file=self.stdout) logging.debug("Exception while enabling a debug hookup " + line) logging.info("Enabled a debug hookup " + line) def do_debug_delete(self, line): """ Delete an existing enabled or disabled debug hookup """ try: del self.dict['debug_infrastructure']['disabled_hookups'][line] print >>self.stdout, "Deleted the debug hookup from disabled" logging.info("Deleted the debug hookup from disabled") except: pass try: del self.dict['debug_infrastructure']['enabled_hookups'][line] print >>self.stdout, "Deleted the debug hookup from enabled" logging.info("Deleted the debug hookup from enabled") except: pass def do_debug_list_enabled(self, line): """ List existing enabled debug hookups """ self.do_ls("/debug_infrastructure/enabled_hookups", quiet=True) def do_debug_list_disabled(self, line): """ List existing disabled debug hookups """ self.do_ls("/debug_infrastructure/disabled_hookups", quiet=True) def do_debug_list(self, line): """ List existing debug hookups """ print >>self.stdout, "--Enabled-------------------------------------------------------------------" self.do_debug_list_enabled(line) print >>self.stdout, "--Disabled------------------------------------------------------------------" self.do_debug_list_disabled(line) def do_quit(self, line): """ Exit program """ return True def list_to_dict(self, something): if isinstance(something, dict): return something.copy() if isinstance(something, list): d = {} for i in xrange(len(something)): d["EL_"+str(i)] = self.list_to_dict(something[i]) return d def do_python(self, line): """ Execute a python command in the current context. Objects appearing in the current context will be accesible by their name. One can put import statements etc. :param line: :return: """ oldout = sys.stdout olderr = sys.stderr env = self.list_to_dict(self.current_element) sys.stdout=self.stdout sys.stderr=self.stdout try: exec(line, env, env) except: traceback.print_exc(file=self.stdout) sys.stdout = oldout sys.stderr = olderr def help_quit(self): print >>self.stdout, "Exit the interactive shell" do_EOF = do_quit help_EOF = help_quit def search_recursive(self, dict_object, path, name, leaf_method): if (isinstance(dict_object, np.ndarray) or isinstance(dict_object, SharedArray.SharedNumpyArray) or isinstance(dict_object, SharedArray.DoubleBufferedSharedNumpyArray)): if self.filter_name is not None: if name == self.filter_name: leaf_method(path, name, dict_object) else: leaf_method(path, name, dict_object) return if isinstance(dict_object, dict): for key in dict_object.keys(): self.search_recursive(dict_object[key], path + "/" + str(key), key, leaf_method) if isinstance(dict_object, list): for i, element in enumerate(dict_object): self.search_recursive(element, path + "[" +str(i) + "]", "", leaf_method) def print_path(self, path, name, element): print >>self.stdout, path def check_gt(self, path, name, element): if np.max(element) > self.gt: print >>self.stdout, path + " has elements greater than " + str(self.gt) + " (" + str(np.max(element)) +")" print >>self.stdout, element def check_abs(self, path, name, element): if np.max(np.fabs(element)) > self.abss: print >>self.stdout, path + " has elements with absolute value greater than " + str(self.abss) + " (" + str(np.max(np.fabs(element))) +")" print >>self.stdout, element def check_lt(self, path, name, element): if np.min(element) < self.lt: print >>self.stdout, path + " has elements less than " + str(self.lt) + " (" + str(np.min(element)) +")" print >>self.stdout, element def check_nans(self, path, name, element): if np.isnan(element).any(): print >>self.stdout, path + " contains NANs " print >>self.stdout, element def check_id(self, path, name, element): if int(id(element)) == self.search_identity: print >>self.stdout, path def run_noninteractive(self): if self.pprint: pprint.pprint(self.dict, stream=self.stdout) if self.sprint: self.search_recursive(self.dict, self.filename, "", self.print_path) if self.gt is not None: self.search_recursive(self.dict, self.filename, "", self.check_gt) if self.lt is not None: self.search_recursive(self.dict, self.filename, "", self.check_lt) if self.nans is not None: self.search_recursive(self.dict, self.filename, "", self.check_nans) if self.abss is not None: self.search_recursive(self.dict, self.filename, "", self.check_abs) def do_nanscheck(self, line): """ Find all elements in the current subtree that contain NANs """ self.search_recursive(self.current_element, self.pwd, "", self.check_nans) def do_gtcheck(self, line): """ Find all elements in the current subtree that contain values greater than a given value Usage: gtcheck value where value is a numeric value """ try: self.gt = float(line) self.search_recursive(self.current_element, self.pwd, "", self.check_gt) except: print >>self.stdout, "Error executing a command" self.do_help("gtcheck") def do_ltcheck(self, line): """ Find all elements in the current subtree that contain values less than a given value Usage: ltcheck value where value is a numeric value """ try: self.lt = float(line) self.search_recursive(self.current_element, self.pwd, "", self.check_lt) except: print >>self.stdout, "Error executing a command" self.do_help("ltcheck") def do_abscheck(self, line): """ Find all elements in the current subtree that contain values greater in absolute value than a given value. Usage: abscheck value where value is a numeric value """ try: self.abss = float(line) self.search_recursive(self.current_element, self.pwd, "", self.check_abs) except: print >>self.stdout, "Error executing a command" self.do_help("abscheck") def do_findid(self, line): """ Find all the occurrences of an object with a given id. Usage: findid value where value is the numeric id """ try: self.search_identity = int(line) self.search_recursive(self.dict, self.filename, "", self.check_id) except: print >>self.stdout, "Error executing a command" self.do_help("findid") def do_findrefs(self, line): """ Finds all the references to a given object inside of the whole structure. Usage: findrefs object where object is in the current scope. """ try: if line.isdigit(): element = self.current_element[int(line)] else: element = self.current_element[line] self.search_identity = int(id(element)) self.search_recursive(self.dict, self.filename, "", self.check_id) except: print >>self.stdout, "Error executing a command" self.do_help("findrefs") def do_pause(self, line): """ Set the paused flag of a simulation (works when connected to a running simulation) """ self.dict['paused'][0] = PVM_Create.PVM_PAUSE def do_step(self, line): """ Stepwise execution. Be careful, works when connected to a runing simulation, otherwise will hang. """ N = self.dict['N'][0] self.dict['paused'][0] = PVM_Create.PVM_RESUME while self.dict['N'][0] <= (N+1): pass self.dict['paused'][0] = PVM_Create.PVM_PAUSE def do_finish(self, line): """ Set the paused flag of a simulation (works when connected to a running simulation) """ self.dict['finished'][0] = PVM_Create.PVM_FLAG_VAL_TRIGGER return True def do_toggle_display(self, line): """ Changes the logic for displaying a window (works when connected to a running simulation) """ self.dict['finished'][0] = PVM_Create.PVM_FLAG_TRIGGER_DISPLAY def do_record(self, line): """ Start recording to a file given after space, e.g.: record myfilename.avi """ if line == "": print >>self.stdout, "No file name given" else: name = np.fromstring(line, dtype=np.uint8) self.dict["record_filename"][:] = 0 self.dict["record_filename"][:name.shape[0]] = name self.dict['finished'][0] = PVM_Create.PVM_FLAG_TRIGGER_RECORD print >>self.stdout, "Started recording" def do_stop_record(self, line): """ Stop recording to a file """ self.dict['finished'][0] = PVM_Create.PVM_FLAG_TRIGGER_RECORD print >>self.stdout, "Stopped recording" def do_toggle_dream(self, line): """ Sets the flags to inform the manager to run the dream mode in which predicted input is fed back as new input """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_DREAM else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_blindspot(self, line): """ Sets the flags to inform the manager to show a gray spot in the center of the image """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_BLINDSPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_gray(self, line): """ Set neutral gray as input """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_GRAY else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_noise(self, line): """ Set white noise as input """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_NOISE else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_noise_spot(self, line): """ Set white noise spot in the center of the image """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_NOISE_SPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_inverse_spot(self, line): """ Set inverse blind spot (gray out periphery) """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_INV_BLINDSPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_inverse_spot_noise(self, line): """ Set inverse blind spot noise (noise out periphery) """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_INV_NOISE_SPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_partial_dream(self, line): """ Set dream mode in the periphery of the visual field """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_INV_DREAM_SPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_dream_spot(self, line): """ Set dream mode in the periphery of the visual field """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_DREAM_SPOT else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_deep_dream(self, line): """ Make all the units in the hierachy dream """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_DEEP_DREAM else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_blinks(self, line): """ Set dream mode in the periphery of the visual field """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_BLINKS else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_toggle_noisy_signal(self, line): """ Set dream mode in the periphery of the visual field """ if self.dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_RESET: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_NOISY_SIGNAL else: self.dict['flags'][0] = PVM_Create.PVM_FLAG_VAL_RESET def do_disable_lateral(self, line): """ Remove the lateral communication between the units """ self.dict['context_factor_lateral'][0] = 0 def do_disable_feedback(self, line): """ Remove the feedback communication between the units """ self.dict['context_factor_feedback'][0] = 0 def do_enable_lateral(self, line): """ Remove the lateral communication between the units """ self.dict['context_factor_lateral'][0] = 1.0 def do_enable_feedback(self, line): """ Remove the feedback communication between the units """ self.dict['context_factor_feedback'][0] = 1.0 def do_freeze_learning(self, line): """ Sets the flags to inform the manager to run the dream mode in which predicted input is fed back as new input """ self.dict['flags'][PVM_Create.PVM_LEARNING_FLAG] = PVM_Create.PVM_LEARNING_FREEZE def do_unfreeze_learning(self, line): """ Sets the flags to inform the manager to run the dream mode in which predicted input is fed back as new input """ self.dict['flags'][PVM_Create.PVM_LEARNING_FLAG] = PVM_Create.PVM_LEARNING_UNFREEZE def do_resume(self, line): """ Set the paused flag of a simulation (works when connected to a running simulation) """ self.dict['paused'][0] = PVM_Create.PVM_RESUME def do_dump(self, line): """ Dump the state of the current simulation to a given file. Warning, this method will attempt to pause a simulation and wait 1s, but is not guaranteed to save a consistent state. Use only in an emergency. """ self.dict['paused'][0] = PVM_Create.PVM_PAUSE time.sleep(1) import PVM_framework.CoreUtils as CoreUtils CoreUtils.save_model(self.dict, line) self.dict['paused'][0] = PVM_Create.PVM_RESUME if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('input_file', default='demo.p.gz', nargs=1, help='Input file dictionary') parser.add_argument("-p", "--pprint", help="Print the contents of the entire dictionary", action="store_true") parser.add_argument("-s", "--sprint", help="Print all the paths in the dictionary", action="store_true") parser.add_argument("-n", "--nans", help="Search for NANs", action="store_true") parser.add_argument("-u", "--upgrade", help="Upgrade to ver 1", action="store_true") parser.add_argument("-g", "--gt", type=str, help="Search for values greater than ") parser.add_argument("-l", "--lt", type=str, help="Search for values less than ") parser.add_argument("-a", "--abs", type=str, help="Search for values with absol;ute value greater than ") parser.add_argument("-f", "--filter", type=str, help="Display only filtered the elements") parser.add_argument("-i", "--interactive", help="Start a shell like command line interface to explore the dictionary", action="store_true") args = parser.parse_args() if not args.input_file: parser.print_help() else: app = InteractiveDictionaryExplorer(infilename=args.input_file[0], pprint=args.pprint, sprint=args.sprint, gt=args.gt, lt=args.lt, nans=args.nans, abss=args.abs, filter_name=args.filter, upgrade=args.upgrade) if args.interactive: app.cmdloop() else: app.run_noninteractive() ================================================ FILE: PVM_framework/PVM_display_helper.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import cv2 import logging import os class VideoRecorder(object): def __init__(self, rec_filename): """ :param rec_filename: :return: Handy object to carry out video recording """ if rec_filename[-4:] == ".avi": rec_filename = rec_filename[:-4] rec_filename = rec_filename + "_%02d.avi" self.rec_filename = rec_filename self._video = None self.index = 0 def set_filename(self, filename): self.override_name = filename def _get_filename(self): if self.override_name is None: return self.rec_filename % self.index else: return self.override_name def record(self, image): """ :param image: :return: Takes and image and records it into a file """ if self._video is None: self._video = cv2.VideoWriter() fps = 20 retval = self._video.open(os.path.expanduser(self._get_filename()), cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), fps, (image.shape[1], image.shape[0])) assert(retval) logging.info("Creating an avi file %s" % os.path.expanduser(self._get_filename())) self._video.write(image) def finish(self): """ When done releases the cv video writer :return: """ if self._video is not None: self._video.release() self._video = None self.index += 1 logging.info("Finished recording file %s" % os.path.expanduser(self._get_filename())) class DisplayHelperObject(object): """ An object that allows to easily assemble display frame """ def __init__(self, width=1920, height=1080, margin=8, grid=(4, 5)): self.frame = np.zeros((height, width, 3), dtype=np.uint8) self.margin = margin self.im_size = min((width-2*self.margin)/grid[0], (height-2*self.margin)/grid[1])-2*self.margin self.grid = grid self.clear_frame() self.logo = cv2.imread(os.path.join(os.path.dirname(os.path.abspath(__file__)), "bc_logo_gray_sm.png")) self.no_refresh = {} def grid_to_pix(self, x, y): if x < 0 or x > self.grid[0] or y < 0 or y > self.grid[1]: raise Exception("Out of grid!") px = x*(self.im_size+2*self.margin)+2*self.margin py = y*(self.im_size+2*self.margin)+2*self.margin return py, px def place_gray_image(self, grid_x, grid_y, image, text=""): (px, py) = self.grid_to_pix(grid_x, grid_y) self.place_label(grid_x=grid_x, grid_y=grid_y, text=text) resized = cv2.resize(image, dsize=(self.im_size, self.im_size), interpolation=cv2.INTER_NEAREST) self.frame[px:px+self.im_size, py:py+self.im_size, 0] = resized self.frame[px:px+self.im_size, py:py+self.im_size, 1] = resized self.frame[px:px+self.im_size, py:py+self.im_size, 2] = resized cv2.rectangle(self.frame, (py-2, px-2), (py+self.im_size+1, px+self.im_size+1), color=(140, 100, 100)) def place_gray_float_image(self, grid_x, grid_y, image, text=""): (px, py) = self.grid_to_pix(grid_x, grid_y) self.place_label(grid_x=grid_x, grid_y=grid_y, text=text) resized = cv2.resize(image, dsize=(self.im_size, self.im_size), interpolation=cv2.INTER_NEAREST) resized *= 255 resized = resized.astype(np.uint8) self.frame[px:px+self.im_size, py:py+self.im_size, 0] = resized self.frame[px:px+self.im_size, py:py+self.im_size, 1] = resized self.frame[px:px+self.im_size, py:py+self.im_size, 2] = resized cv2.rectangle(self.frame, (py-2, px-2), (py+self.im_size+1, px+self.im_size+1), color=(140, 100, 100)) def place_color_image(self, grid_x, grid_y, image, text=""): (px, py) = self.grid_to_pix(grid_x, grid_y) self.place_label(grid_x=grid_x, grid_y=grid_y, text=text) cv2.rectangle(self.frame, (py-2, px-2), (py+self.im_size+1, px+self.im_size+1), color=(140, 100, 100)) self.frame[px:px+self.im_size, py:py+self.im_size, :] = cv2.resize(image, dsize=(self.im_size, self.im_size), interpolation=cv2.INTER_NEAREST) def place_color_float_image(self, grid_x, grid_y, image, text=""): (px, py) = self.grid_to_pix(grid_x, grid_y) self.place_label(grid_x=grid_x, grid_y=grid_y, text=text) cv2.rectangle(self.frame, (py-2, px-2), (py+self.im_size+1, px+self.im_size+1), color=(140, 100, 100)) resized = cv2.resize(image, dsize=(self.im_size, self.im_size), interpolation=cv2.INTER_NEAREST) resized *= 255 resized = resized.astype(np.uint8) self.frame[px:px+self.im_size, py:py+self.im_size, :] = resized def place_image(self, grid_x, grid_y, image, text=""): if image.dtype == np.float: if len(image.shape) == 3 and image.shape[2] == 3: self.place_color_float_image(grid_x, grid_y, image, text) else: self.place_gray_float_image(grid_x, grid_y, image, text) else: if len(image.shape) == 3 and image.shape[2] == 3: self.place_color_image(grid_x, grid_y, image, text) else: self.place_gray_image(grid_x, grid_y, image, text) def place_color_logo(self, grid_x, grid_y, image=None, text=""): if image is None: image = self.logo (px, py) = self.grid_to_pix(grid_x, grid_y) self.frame[px:px+image.shape[0], py:py+image.shape[1], :] = image def place_label(self, grid_x, grid_y, text=""): if not (grid_x, grid_y) in self.no_refresh: (px, py) = self.grid_to_pix(grid_x, grid_y) cv2.putText(self.frame, text, (py+5, px-4), cv2.FONT_HERSHEY_SIMPLEX, 0.3, color=(0, 0, 0), lineType=cv2.CV_AA) self.no_refresh[(grid_x, grid_y)] = 0 def place_text(self, grid_x, grid_y, voffset=0, text=""): (px, py) = self.grid_to_pix(grid_x, grid_y) cv2.putText(self.frame, text, (py, px + voffset), cv2.FONT_HERSHEY_SIMPLEX, 0.3, color=(0, 0, 0), lineType=cv2.CV_AA) def clear_cell(self, grid_x, grid_y): (px, py) = self.grid_to_pix(grid_x, grid_y) self.frame[px:px+self.im_size, py:py+self.im_size, :] = 127 def clear_frame(self): self.frame *= 0 self.frame += 127 self.no_refresh = {} ================================================ FILE: PVM_framework/PVM_options.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import textwrap option_default = { "initial_learning_rate": "0.0003", "final_learning_rate": "0.0001", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "enable_lateral_at": "900000", "enable_feedback_at": "900000", "input_block_size": "5", "hidden_block_size": "5", "layer_shapes": ["4", "3", "2", "1"], "readout_block_size": ["1", "1", "2", "4"], "enable_readout": ["1", "1", "1", "1"], "lateral_radius": "1.5", "context_exclude_self": "0", "fan_in_square_size": "2", "fan_in_radius": "2", "readout_depth": "1", "reverse": "0", "new_name": "", "polynomial": "0", "autoencoder": "0", "model_type": "small", "ex_module": "PVM_models.PVM_unit_v1", "unit_type": "simple", "stereo": "0", "only_one_channel": "0", "supervised": "0", "supervised_rate": "0.0002", "steps": "100000000", "bias_free": "0", "bias_free_readout": "0", "disable_lateral": "0", "disable_feedback": "0", "dataset": "short", "last_layer_context_to_all": "1", "send_context_two_layers_back": "0", "version_major": "0", "version_minor": "0", "use_t_minus_2_block": "0", "use_derivative": "1", "use_integral": "1", "use_error": "1", "predict_two_steps": "0", "use_global_backprop": "0", "feed_context_in_complex_layer": "0", "tau": "0.98", "momentum": "0.5", "normalize_output": "0", "save_source_files": "0", "backpropagate_readout_error": "0", } option_descriptions = { "initial_learning_rate": "Learning rate used in the initial stage of the simulation.", "final_learning_rate": "Learning rate used in the remaining part of the simulation.", "delay_each_layer_learning": "Enable learning in consequtive layers by this number of steps.", "delay_final_learning_rate": "Switch to final learning rate by this number of steps.", "enable_lateral_at": "Enable lateral connections at this step.", "enable_feedback_at": "Enable feedback at this step.", "input_block_size": "Size of the input tile.", "hidden_block_size": "Size of the internal representation.", "layer_shapes": "List of layer sizes from the input layer towards higher layers.", "readout_block_size": "Dimensions of he readout blocks in each layer.", "enable_readout": "Determines in which layers the readout is performed.", "lateral_radius": "Radius of the lateral interactions, 1 - 4 neighbourhood, 1.5 - 8 neighbourhood, 2 - 12 neighbourhood. See PVM_Create.py for details.", "context_exclude_self": "Exclude self as a source of context.", "fan_in_square_size": "Size of the square from which the fan ins will be selected. See PVM_Create for details.", "fan_in_radius": "Radius in within which the fan ins will be selected.", "readout_depth": "Depth of the readout classifier.", "reverse": "Run the input dataset in reverse order.", "new_name": "Set the new name for a simulation, subsequet snapshots will be saved in a new S3 directory.", "polynomial": "Use the polynomial sigmoid function in the MLP (slightly faster computation).", "autoencoder": "For executions modules that allow, use autoencoder instead of predictive encoder, supported by PVM_unit.", "model_type": "The architecture name, choose from tiny, small2, small and large.", "ex_module": "Name of the python file (without .py) that will be used as the execution unit e.g. PVM_unit.", "unit_type": "For execution modules that support choose from simple or complex. Simple is typically a 3 layer MLP, complex is typically 4 layer MLP.", "stereo": "Use an interlaced stereo video for input.", "only_one_channel": "Use only one channel from a stereo dataset (interlaced with black frame).", "supervised": "Run the simulation in supervised mode.", "supervised_rate": "Learning rate for the readout classifier.", "steps": "Number of steps to run for.", "bias_free": "Use a biasfree perceptron (with bias unit equal zero).", "bias_free_readout": "Use bias free perceptron for the readout classifier.", "disable_lateral": "Do not enable lateral context.", "disable_feedback": "Do not enable feedback context.", "dataset": "Name of the dataset to use.", "last_layer_context_to_all": "Send last layer activations as context to every layer below.", "send_context_two_layers_back": "Send context two layers backwards in addition to normal context one layer backwards.", "version_major": "Major version number of the dictionary.", "version_minor": "Minor version number of the dictionary.", "use_t_minus_2_block": "Use t-2 in addition to t-1 as predictor input, supported by PVM_unit.", "use_derivative": "Use difference between t-1 and t-2 as additional features, supported by PVM_unit.", "use_integral": "Use integral trace as additional feature with value of time constant given in tau parameter, supported by PVM_unit.", "use_error": "Use error of the previous prediction as additional feature, supported by PVM_unit.", "predict_two_steps": "Predict two steps in the future, supported by PVM_unit.", "use_global_backprop": "Backpropagate error through context and feedforward connections outside of the unit, supported by PVM_unit.", "feed_context_in_complex_layer": "Put context activation in the second layer of the complex unit.", "tau": "Time constant used to compute the integral trace feature.", "momentum": "Default momentum value used in the backprop algorithm.", "normalize_output": "Use simple running min-max normalization on the output of the predictive encoder compressed features.", "save_source_files": "Save all *.py and *pyx files from the folder where the simulation was created for future inspection.", "backpropagate_readout_error": "Propagate the error of the readout classifier into the predictive encoder" } option_values = { "initial_learning_rate": "float number", "final_learning_rate": "float number", "delay_each_layer_learning": "int number", "delay_final_learning_rate": "int number", "enable_lateral_at": "int number", "enable_feedback_at": "int number", "input_block_size": "int number", "hidden_block_size": "int number", "layer_shapes": "list of int numbers", "readout_block_size": "list of int numbers", "enable_readout": "list of int numbers", "lateral_radius": "float number", "context_exclude_self": ("0", "1"), "fan_in_square_size": "int number", "fan_in_radius": "int number", "readout_depth": "int number", "reverse": ("0", "1"), "new_name": "str", "polynomial": ("0", "1"), "autoencoder": ("0", "1"), "model_type": ("tiny", "small2", "small", "large"), "ex_module": ("PVM_unit", "PVM_unit_2step", "PVM_unit_2step_residual"), "unit_type": ("simple", "complex"), "stereo": ("0", "1"), "only_one_channel": ("0", "1"), "supervised": ("0", "1"), "supervised_rate": "float number", "steps": "int number", "bias_free": ("0", "1"), "bias_free_readout": ("0", "1"), "disable_lateral": ("0", "1"), "disable_feedback": ("0", "1"), "dataset": "str", "last_layer_context_to_all": ("0", "1"), "send_context_two_layers_back": ("0", "1"), "version_major": "int number", "version_minor": "int number", "use_t_minus_2_block": ("0", "1"), "use_derivative": ("0", "1"), "use_integral": ("0", "1"), "use_error": ("0", "1"), "predict_two_steps": ("0", "1"), "use_global_backprop": ("0", "1"), "feed_context_in_complex_layer": ("0", "1"), "tau": "float number", "momentum": "float number", "normalize_output": ("0", "1"), "save_source_files": ("0", "1"), "backpropagate_readout_error": ("0", "1"), } def get_option_help(): wr = textwrap.TextWrapper(initial_indent=' ', subsequent_indent=' ', width=70) help_str = "Available options (given as json dict either from the command line -O or in a file via -S):\n" help_str += "\n" for option in option_default.keys(): option_str = "" option_str += " %s - default value %s. " % (option, option_default[option]) option_str += " %s Possible values: " % (option_descriptions[option]) if type(option_values[option]) == tuple: for vals in option_values[option]: option_str += "\"%s\" " % vals else: option_str += option_values[option] + "\n" help_str += wr.fill(option_str) + "\n" return help_str def parse_options(options_given, options_in_the_dict=None): parsed = option_default.copy() for option in option_default.keys(): if options_in_the_dict is not None and option in options_in_the_dict: parsed[option] = options_in_the_dict[option] if option in options_given: parsed[option] = options_given[option] for option in options_given.keys(): if option not in option_default.keys(): if option == "bias_free_additional": parsed['bias_free_readout'] = options_given["bias_free_additional"] else: raise Exception("Unknown option %s given" % option) return parsed ================================================ FILE: PVM_framework/PVM_tracker.json ================================================ { "nodes": [ { "name": "camera", "params": {"camera_index": 0, "desired_width": 160, "desired_height": 120}, "loader": "python:BC.brainos_library.node.environments.universal_camera_input_node.UniversalCameraInputNode" }, { "name": "viewer", "params": {"window_name": "SaliencyTrackerNode View - Drag and Drop to Prime Tracker", "desired_width": 320, "desired_height": 240}, "loader": "python:BC.brainos_library.node.environments.tracker_image_viewer_node.TrackerImageViewerNode" }, { "name": "tracker", "params": {"remote_filename": "DARPA/Simulations/2015_11_18_15_04_20_context_in_middle_smaller_b434b5f3/PVM_failsafe_0002900000.p.gz", "cores": 20}, "loader": "python:future_encoder_framework.PVM_node.PVMTrackerNode" } ], "graph": [ { "signal": "camera/image-out", "slot": "viewer/image-in" }, { "signal": "camera/image-out", "slot": "tracker/image-in" }, { "signal": "tracker/box-out", "slot": "viewer/box-in" }, { "signal": "viewer/box-out", "slot": "tracker/box-in" } ] } ================================================ FILE: PVM_framework/PVM_upgrade_dict.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import os import logging import argparse import PVM_framework.CoreUtils as CoreUtils import PVM_framework.PVM_Create as PVM_Create import PVM_framework.PVM_Storage as PVM_Storage if __name__ == '__main__': logging.basicConfig(filename="PVM_upgrade.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(thread)d PVM_run : %(message)s ') logging.getLogger().addHandler(logging.StreamHandler()) logging.info("###################################################################") logging.info(" STARTING NEW RUN ") logging.info("###################################################################") parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="File to load", type=str, default="") parser.add_argument("-r", "--remote", help="Download and run a remote simulation", type=str, default="") parser.add_argument("-d", "--destination", help="Where to save the model", type=str, default="PVM_models/") parser.add_argument("-n", "--name", help="New name", type=str, default="") args = parser.parse_args() Storage = PVM_Storage.Storage() if args.remote != "": filename = Storage.get(args.remote) logging.info("Loaded a remote simulation dict %s" % args.remote) if os.path.isfile(filename): simulation_dict = CoreUtils.load_model(filename) logging.info("Loaded the dictionary") PVM_Create.upgrade_dictionary_to_ver1_0(simulation_dict) for k in sorted(simulation_dict.keys()): print k if args.name != "": simulation_dict['name'] = args.name CoreUtils.save_model(simulation_dict, "PVM_failsafe_%010d.p.gz" % int(simulation_dict['N'][0])) to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) from_path = "./PVM_failsafe_%010d.p.gz" % int(simulation_dict['N'][0]) logging.info("Uploading %s/%s" % (to_folder, from_path[2:])) Storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) ================================================ FILE: PVM_framework/SharedArray.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np from multiprocessing.sharedctypes import RawArray import sys import mmap import ctypes import posix_ipc from _multiprocessing import address_of_buffer from string import ascii_letters, digits import pickle valid_chars = frozenset("-_. %s%s" % (ascii_letters, digits)) class ShmemBufferWrapper(object): """ IPC shared memory buffer wrapper. """ def __init__(self, tag, size, create=True): self._mem = None self._map = None self._owner = create self.size = size assert 0 <= size < sys.maxint flag = (0, posix_ipc.O_CREX)[create] if create: self._mem = posix_ipc.SharedMemory(tag, flags=flag, size=size) else: self._mem = posix_ipc.SharedMemory(tag, flags=flag, size=0) self._map = mmap.mmap(self._mem.fd, self._mem.size) self._mem.close_fd() def get_address(self): addr, size = address_of_buffer(self._map) assert size >= self.size return addr def __del__(self): if self._map is not None: self._map.close() if self._mem is not None and self._owner: self._mem.unlink() def ShmemRawArray(type_, size_or_initializer, tag, create=True): """ Raw shared memory array based on IPC tag :param type_: :param size_or_initializer: :param tag: :param create: :return: """ if tag[0] != "/": tag = "/%s" % (tag,) if isinstance(size_or_initializer, int): type_ = type_ * size_or_initializer else: type_ = type_ * len(size_or_initializer) buffer = ShmemBufferWrapper(tag, ctypes.sizeof(type_), create=create) obj = type_.from_address(buffer.get_address()) obj._buffer = buffer if not isinstance(size_or_initializer, int): obj.__init__(*size_or_initializer) return obj def np_type_id_to_ctypes(dtype): type_id = None if hasattr(np, 'float') and dtype == np.float: type_id = ctypes.c_int64 return type_id if hasattr(np, 'float16') and dtype == np.float16: type_id = ctypes.c_int16 return type_id if hasattr(np, 'float32') and dtype == np.float32: type_id = ctypes.c_int32 return type_id if hasattr(np, 'float64') and dtype == np.float64: type_id = ctypes.c_int64 return type_id if hasattr(np, 'float128') and dtype == np.float128: type_id = (ctypes.c_int64 * 2) return type_id if hasattr(np, 'int') and dtype == np.int: type_id = ctypes.c_int64 return type_id if hasattr(np, 'uint8') and dtype == np.uint8: type_id = ctypes.c_byte return type_id if hasattr(np, 'int8') and dtype == np.int8: type_id = ctypes.c_byte return type_id if hasattr(np, 'uint16') and dtype == np.uint16: type_id = ctypes.c_int16 return type_id if hasattr(np, 'int16') and dtype == np.int16: type_id = ctypes.c_int16 return type_id if hasattr(np, 'uint32') and dtype == np.uint32: type_id = ctypes.c_int32 return type_id if hasattr(np, 'int32') and dtype == np.int32: type_id = ctypes.c_int32 return type_id if hasattr(np, 'uint64') and dtype == np.uint64: type_id = ctypes.c_int64 return type_id if hasattr(np, 'int64') and dtype == np.int64: type_id = ctypes.c_int64 return type_id if hasattr(np, 'complex') and dtype == np.complex: type_id = (ctypes.c_int64 * 2) return type_id if hasattr(np, 'complex64') and dtype == np.complex64: type_id = (ctypes.c_int64) return type_id if hasattr(np, 'intc') and dtype == np.intc: type_id = (ctypes.c_int) return type_id if hasattr(np, 'intp') and dtype == np.intp: type_id = (ctypes.c_ssize_t) return type_id if hasattr(np, 'bool') and dtype == np.bool: type_id = (ctypes.c_byte) return type_id raise Exception('No matching data type!') class SharedNumpyArray: # DO NOT EXTEND FROM OBJECT! """ Acts like a numpy array but is shared! One notable difference - added copyto method which copies from a given object and places the content in the shared memory. The reason for that is that numpy.copyto will not work on this object directly unfortunately. Uses raw array so the if you need a lock you have to instantiate it yourself. If given a tag in the constructor, will actually generate a POSIX IPC memory that can be attached from a different process. Note that if this is the case, the array is no longer serializable. """ def __init__(self, shape, dtype, tag=None, create=True): type_id = np_type_id_to_ctypes(dtype) self.tag = tag if tag is not None: self.__shared = ShmemRawArray(type_id, np.product(shape), tag, create=create) else: self.__shared = RawArray(type_id, np.product(shape)) self.__np_array = np.frombuffer(self.__shared, dtype=dtype).reshape(shape) def __getattr__(self, name): """ This is only called if python cannot find the attribute. This is the reason why this class cannot be derived from object which comes with a bunch of attributes and screws this thing up. """ return self.__np_array.__getattribute__(name) def __getstate__(self): """ Method overloaded for support of pickling. """ if self.tag is not None: raise pickle.PicklingError() state = self.__dict__.copy() del state['_SharedNumpyArray__shared'] return state def __setstate__(self, state): """ Method overloaded for support of pickling. """ shape = state['_SharedNumpyArray__np_array'].shape dtype = state['_SharedNumpyArray__np_array'].dtype type_id = np_type_id_to_ctypes(dtype) self.__shared = RawArray(type_id, np.product(shape)) self.__np_array = np.frombuffer(self.__shared, dtype=dtype).reshape(shape) np.copyto(self.__np_array, state['_SharedNumpyArray__np_array']) self.tag = None def copyto(self, nparray): """ Implemented because numpy.copyto will not work directly on this object """ np.copyto(self.__np_array, nparray) class DynamicView: """ Picklable view object. A view similar to a pointer offers an array like object whose content coresponds to a subarray of some existing array. Normal views on shared numpy arrays are not picklable (after unpickling they will recrate a copy of the original array). Dynamic view is fully picklable. """ def __init__(self, array): self._array = array self._view = self._array self._view_item = None self._version = 0.1 def __getitem__(self, item): if type(item) == slice or (type(item) == tuple) or type(item) == int: self._view_item = item self._view = self._array.__getitem__(item) return self def __getattr__(self, name): """ This is only called if python cannot find the attribute. This is the reason why this class cannot be derived from object which comes with a bunch of attributes and screws this thing up. """ return self._view.__getattribute__(name) def __getstate__(self): """ Method overloaded for support of pickling. """ state = self.__dict__.copy() del state['_view'] return state def __setstate__(self, state): """ Method overloaded for support of pickling. """ if "_version" not in state.keys(): # Compatibility mode self._array = state['_array'] self._version = 0.1 # promote to the latest version x = state['_x'] y = state['_y'] z = state['_z'] dx = state['_dx'] dy = state['_dy'] dz = state['_dz'] if x is not None and y is not None and z is not None: self[x:x+dx, y:y+dy, z:z+dz] elif x is not None and y is not None: self[x:x+dx, y:y+dy] elif x is not None: self[x:x+dx] else: self._view = self._array self._view_item = None else: self.__dict__ = state if self._view_item is not None: self.__getitem__(self._view_item) else: self._view = self._array def SharedNumpyArray_like(array): """ Creates a shared array object identical as the given numpy array (but without the content) :param array: numpy array object :return: shared array object """ return SharedNumpyArray(shape=array.shape, dtype=array.dtype) class DoubleBufferedSharedNumpyArray: # DO NOT EXTEND FROM OBJECT! """ Acts like a numpy array but is shared! One notable difference - added copyto method which copies from a given object and places the content in the shared memory. The reason for that is that numpy.copyto will not work on this object directly unfortunately. Uses raw array so the if you need a lock you have to instantiate it yourself. """ def __init__(self, shape, dtype, parity_obj): type_id = np_type_id_to_ctypes(dtype) self.__shared1 = RawArray(type_id, np.product(shape)) self.__np_array1 = np.frombuffer(self.__shared1, dtype=dtype).reshape(shape) self.__shared2 = RawArray(type_id, np.product(shape)) self.__np_array2 = np.frombuffer(self.__shared2, dtype=dtype).reshape(shape) self.__parity = parity_obj def __getattr__(self, name): if self.__parity[0]==0: return self.__np_array1.__getattribute__(name) else: return self.__np_array2.__getattribute__(name) def __getstate__(self): """ Method overloaded for support of pickling. """ state = self.__dict__.copy() del state['_DoubleBufferedSharedNumpyArray__shared1'] del state['_DoubleBufferedSharedNumpyArray__shared2'] return state def __setstate__(self, state): """ Method overloaded for support of pickling. """ shape = state['_DoubleBufferedSharedNumpyArray__np_array1'].shape dtype = state['_DoubleBufferedSharedNumpyArray__np_array1'].dtype type_id = np_type_id_to_ctypes(dtype) self.__shared1 = RawArray(type_id, np.product(shape)) self.__np_array1 = np.frombuffer(self.__shared1, dtype=dtype).reshape(shape) np.copyto(self.__np_array1, state['_DoubleBufferedSharedNumpyArray__np_array1']) self.__shared2 = RawArray(type_id, np.product(shape)) self.__np_array2 = np.frombuffer(self.__shared2, dtype=dtype).reshape(shape) np.copyto(self.__np_array2, state['_DoubleBufferedSharedNumpyArray__np_array2']) self.__parity = state['_DoubleBufferedSharedNumpyArray__parity'] def copyto(self, nparray): """ Implemented because numpy.copyto will not work directly on this object """ if self.__parity[0]==0: np.copyto(self.__np_array1, nparray) else: np.copyto(self.__np_array2, nparray) def copytobuffer(self, nparray): """ Implemented because numpy.copyto will not work directly on this object """ if self.__parity[0]==0: np.copyto(self.__np_array2, nparray) else: np.copyto(self.__np_array1, nparray) ================================================ FILE: PVM_framework/Sync.cpp ================================================ /*# ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== */ #include "Sync.h" long atomic_get_and_set(long n, long *addr) { return __atomic_exchange_n(addr, n, __ATOMIC_SEQ_CST); } void acquire(long *addr) { while (atomic_get_and_set(1, addr) != 0); } void release(long *addr) { atomic_get_and_set(0, addr); } ================================================ FILE: PVM_framework/Sync.h ================================================ /* # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== */ long atomic_get_and_set(long, long*); void acquire(long *); void release(long *); ================================================ FILE: PVM_framework/SyncUtils.pyx ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cython from multiprocessing.sharedctypes import RawArray import ctypes from cpython.buffer cimport PyBUF_SIMPLE from cpython.buffer cimport Py_buffer from cpython.buffer cimport PyObject_GetBuffer from cpython.buffer cimport PyBuffer_Release import time import multiprocessing as mp cdef extern from "Sync.h": cdef long atomic_get_and_set(long n, long*) cdef void acquire(long*) cdef void release(long*) cdef void flush_denormals() cdef class SpinLock: cdef long* addr cdef int key cdef object array def __init__(self): self.array = RawArray(ctypes.c_int64, 1) cdef Py_buffer view PyObject_GetBuffer(self.array, &view, PyBUF_SIMPLE) self.addr = view.buf def acquire(self): acquire(self.addr) def release(self): release(self.addr) def __enter__(self): return self.acquire() def __exit__(self, dtype, value, traceback): return self.release() cdef class SpinLockRaw: cdef long* addr cdef int key cdef object array def __init__(self): self.array = RawArray(ctypes.c_int64, 1) cdef Py_buffer view PyObject_GetBuffer(self.array, &view, PyBUF_SIMPLE) self.addr = view.buf def acquire(self): acquire(self.addr) def release(self): release(self.addr) def __enter__(self): return self.acquire() def __exit__(self, dtype, value, traceback): return self.release() cdef class Barrier: """ Object meant to provide barrier functionality. Instantiated in the parent process, can be passed onto child processes and used for synchronization. Relies on multiprocessing Value and Lock. """ cdef object worker_procs cdef object workers_waiting cdef object workers_ready cdef object parent_resumed cdef object __quit_workers cdef object worker_procs_lock cdef object workers_waiting_lock cdef object workers_ready_lock cdef object __timeout def __init__(self, num_proc, timeout = 0, UseSpinLock=True): self.worker_procs = mp.sharedctypes.RawValue('i') self.worker_procs.value = num_proc self.workers_waiting = mp.sharedctypes.RawValue('i') self.workers_ready = mp.sharedctypes.RawValue('i') self.parent_resumed = mp.sharedctypes.RawValue('i') self.__quit_workers = mp.sharedctypes.RawValue('i') if UseSpinLock: self.worker_procs_lock = SpinLock() self.workers_waiting_lock = SpinLock() self.workers_ready_lock = SpinLock() else: self.worker_procs_lock = mp.Lock() self.workers_waiting_lock = mp.Lock() self.workers_ready_lock = mp.Lock() self.__timeout = timeout def __inc(self, val, lock): lock.acquire() val.value+=1 lock.release() def __check_and_reset(self, val, lock, check_val): if val.value == check_val: lock.acquire() val.value = 0 lock.release() return True return False def worker_barrier(self): """ The call will block the execution on the worker side until all the workers reach barrier_worker and the parent process reaches resume_workers """ self.__inc(self.workers_waiting, self.workers_waiting_lock) while self.workers_waiting.value != 0: if self.__timeout>0: time.sleep(self.__timeout) while self.parent_resumed.value == 0: if self.__timeout>0: time.sleep(self.__timeout) if self.__quit_workers.value == 1: self.worker_procs_lock.acquire() self.worker_procs.value-=1 self.worker_procs_lock.release() return False self.__inc(self.workers_ready, self.workers_ready_lock) while self.workers_ready.value != 0: if self.__timeout>0: time.sleep(self.__timeout) if self.__quit_workers.value == 1: self.worker_procs_lock.acquire() self.worker_procs.value-=1 self.worker_procs_lock.release() return False return True def parent_barrier(self, quit_workers=False): """ Wait until all the workers reach barrier_worker. Resume once they do. """ self.parent_resumed.value = 0 while not self.__check_and_reset(self.workers_waiting, self.workers_waiting_lock, self.worker_procs.value): if self.__timeout>0: time.sleep(self.__timeout) if quit_workers: self.__quit_workers.value = 1 self.parent_resumed.value = 1 def resume_workers(self, quit_workers=False): """ Notify the worker processes to resume from barrier_worker """ if quit_workers: self.__quit_workers.value = 1 while not self.__check_and_reset(self.workers_ready, self.workers_ready_lock, self.worker_procs.value): if self.__timeout>0: time.sleep(self.__timeout) def quit_workers(self): self.parent_barrier(quit_workers=True) def workers_running(self): """ Return the number of active workers """ return self.worker_procs.value ================================================ FILE: PVM_framework/SyncUtils_python.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time SpinLocksAvailable=True try: import SyncUtils except: SpinLocksAvailable=False class Barrier(): """ Object meant to provide barrier functionality. Instantiated in the parrent process, can be passed onto child processes and used for synchronization. Relies on multiprocessing Value and Lock. """ def __init__(self, num_proc, timeout = 0, UseSpinLock=True): self.worker_procs = mp.sharedctypes.RawValue('i') self.worker_procs.value = num_proc self.workers_waiting = mp.sharedctypes.RawValue('i') self.workers_ready = mp.sharedctypes.RawValue('i') self.parent_resumed = mp.sharedctypes.RawValue('i') self.__quit_workers = mp.sharedctypes.RawValue('i') if UseSpinLock: if not SpinLocksAvailable: raise("Spin locks are not available on this system") self.worker_procs_lock = SyncUtils.SpinLock() self.workers_waiting_lock = SyncUtils.SpinLock() self.workers_ready_lock = SyncUtils.SpinLock() else: self.worker_procs_lock = mp.Lock() self.workers_waiting_lock = mp.Lock() self.workers_ready_lock = mp.Lock() self.__timeout = timeout def __inc(self, val, lock): lock.acquire() val.value+=1 lock.release() def __check_and_reset(self, val, lock, check_val): if val.value == check_val: lock.acquire() val.value = 0 lock.release() return True return False def worker_barrier(self): """ The call will block the execution on the worker side until all the workers reach barrier_worker and the parent process reaches resume_workers """ self.__inc(self.workers_waiting, self.workers_waiting_lock) while self.workers_waiting.value != 0: if self.__timeout>0: time.sleep(self.__timeout) while self.parent_resumed.value == 0: if self.__timeout>0: time.sleep(self.__timeout) if self.__quit_workers.value == 1: self.worker_procs_lock.acquire() self.worker_procs.value-=1 self.worker_procs_lock.release() return False self.__inc(self.workers_ready, self.workers_ready_lock) while self.workers_ready.value != 0: if self.__timeout>0: time.sleep(self.__timeout) if self.__quit_workers.value == 1: self.worker_procs_lock.acquire() self.worker_procs.value-=1 self.worker_procs_lock.release() return False return True def parent_barrier(self, quit_workers=False): """ Wait until all the workers reach barrier_worker. Resume once they do. """ self.parent_resumed.value = 0 while not self.__check_and_reset(self.workers_waiting, self.workers_waiting_lock, self.worker_procs.value): if self.__timeout>0: time.sleep(self.__timeout) if quit_workers: self.__quit_workers.value = 1 self.parent_resumed.value = 1 def resume_workers(self, quit_workers=False): """ Notify the worker processes to resume from barrier_worker """ if quit_workers: self.__quit_workers.value = 1 while not self.__check_and_reset(self.workers_ready, self.workers_ready_lock, self.worker_procs.value): if self.__timeout>0: time.sleep(self.__timeout) def quit_workers(self): self.parent_barrier(quit_workers=True) def workers_running(self): """ Return the number of active workers """ return self.worker_procs.value ================================================ FILE: PVM_framework/__init__.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== ================================================ FILE: PVM_framework/debug_logger.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cPickle import numpy as np class DebugLogger(object): """ Logs data from the simulation going through debug hookups in the dictionary. Each hookup is a structure consisting of: filename - string object - object being dumped interval - integer every how many steps log th data phase - integer (< than interval) depicting phase of when to dump the data path - path of the element being dumped """ def __init__(self, dict, buffer_len=100): """ :param dict: simulation dictionary :type dict: dict :param buffer_len: length of the memory buffer (default 100) :type buffer_len: int :return: No value """ self.dict = dict self.files = {} self.buffers = {} self.buffer_len = buffer_len def process_hookups(self): """ This is called on every step of the simulation when the data is consistent. :return: No value """ current_hookups = [] for hookhash in self.dict['debug_infrastructure']['enabled_hookups'].keys(): hookup = self.dict['debug_infrastructure']['enabled_hookups'][hookhash] current_hookups.append(hookhash) if hookhash not in self.files: file = open(hookhash, "wb") self.files[hookhash] = file if hookhash not in self.buffers: self.buffers[hookhash] = [] if self.dict['N'][0] % hookup['interval'] == hookup['phase']: self.buffers[hookhash].append(hookup['object'].view(np.ndarray).copy()) if len(self.buffers[hookhash])>=self.buffer_len: self.flush(hookhash) for name in self.files.keys(): if name not in current_hookups: self.flush(name) self.files[name].close() del self.files[name] del self.buffers[name] def flush(self, hookhash): """ Empty the buffer and save data o disk :param hookhash: :return: No value """ for object in self.buffers[hookhash]: cPickle.dump(object.view(np.ndarray), self.files[hookhash], protocol=-1) self.buffers[hookhash] = [] def flushall(self): """ Flush all active buffers :return: No value """ for hookhash in self.dict['debug_infrastructure']['enabled_hookups'].keys(): self.flush(hookhash) def __del__(self): self.flushall() for file in self.files.items(): file.close() ================================================ FILE: PVM_framework/fast_routines.pyx ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cython import numpy as np cimport numpy as np from cpython.buffer cimport PyBUF_SIMPLE from cpython.buffer cimport Py_buffer from cpython.buffer cimport PyObject_GetBuffer from cpython.buffer cimport PyBuffer_Release import os from libc.stdlib cimport free from cpython cimport PyObject, Py_INCREF np.import_array() cdef extern from "Accelerated.h": cdef void dot_transpose(double* mult, double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result) cdef void dot_transpose_simple(double* vector, int vect_shape_0, double* matrix, int mat_shape0, int mat_shape1, double* result) cdef void derivative_dot(double* vector, double* vector2, int vect_shape_0, double* result) cdef void derivative_dot_poly(double* vector, double* vector2, int vect_shape_0, double* result) cdef void generalized_outer(double alpha, double * vector1, int vect1_shape, double * vector2, int vect2_shape, double beta, double* matrix, double* result) cdef void dot_sigmoid(double* vector, double* matrix, int mat_shape0, int mat_shape1, double beta, double * result, int append_bias) cdef void dot_sigmoid_poly(double* vector, double* matrix, int mat_shape0, int mat_shape1, double beta, double * result, int append_bias) cdef void dot_add(double* vector, double* matrix, int mat_shape0, int mat_shape1, double * result, int append_bias) cdef void sigmoid_poly(double* result, int shape, double beta) cdef void sigmoid(double* result, int shape, double beta) @cython.boundscheck(False) def derivative_dot_cython(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=1, mode='c'] vector2 not None, np.ndarray[double, ndim=1, mode='c'] result not None): result[:]=vector * (1.0 - vector) * vector2 @cython.boundscheck(False) def derivative_dot_c(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=1, mode='c'] vector2 not None, np.ndarray[double, ndim=1, mode='c'] result not None): cdef int vs0 vs0=vector.shape[0] derivative_dot(& vector[0], & vector2[0], vs0, & result[0]) def derivative_dot_c_poly(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=1, mode='c'] vector2 not None, np.ndarray[double, ndim=1, mode='c'] result not None): cdef int vs0 vs0=vector.shape[0] derivative_dot_poly(& vector[0], & vector2[0], vs0, & result[0]) @cython.boundscheck(False) def dot_transpose_mul_cython(np.ndarray[double, ndim=1, mode='c'] mult not None, np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=1, mode='c'] result not None): result[:]=mult*np.dot(vector, matrix.T) @cython.boundscheck(False) def dot_transpose_cython(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=1, mode='c'] result not None): result[:]=np.dot(vector, matrix.T) @cython.boundscheck(False) def dot_transpose_c(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=1, mode='c'] result not None): cdef int vs0, ms0, ms1 vs0=vector.shape[0] ms0=matrix.shape[0] ms1=matrix.shape[1] dot_transpose_simple(& vector[0], vs0, & matrix[0, 0], ms0, ms1, & result[0]) @cython.boundscheck(False) def dot_transpose_mul_c(np.ndarray[double, ndim=1, mode='c'] mult not None, np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=1, mode='c'] result not None): cdef int vs0, ms0, ms1 cdef double ret vs0=vector.shape[0] ms0=matrix.shape[0] ms1=matrix.shape[1] dot_transpose(& mult[0], & vector[0], vs0, & matrix[0, 0], ms0, ms1, & result[0]) @cython.boundscheck(False) def generalized_outer_cython(double alpha, np.ndarray[double, ndim=1, mode='c'] vector1 not None, np.ndarray[double, ndim=1, mode='c'] vector2 not None, double beta, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=2, mode='c'] result not None ): result[:]=alpha*np.outer(vector1, vector2)+beta*matrix @cython.boundscheck(False) def generalized_outer_c(double alpha, np.ndarray[double, ndim=1, mode='c'] vector1 not None, np.ndarray[double, ndim=1, mode='c'] vector2 not None, double beta, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=2, mode='c'] result not None ): cdef int vs0, vs1 vs0=vector1.shape[0] vs1=vector2.shape[0] generalized_outer(alpha, & vector1[0], vs0, & vector2[0], vs1, beta, & matrix[0, 0], & result[0, 0]) @cython.boundscheck(False) def dot_sigmoid_cython(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, double beta, np.ndarray[double, ndim=1, mode='c'] result not None, int append_bias): if append_bias!=0: vector = np.append(vector, 1.0) result[:]=1.0/(1.0+np.exp(-beta*np.dot(vector, matrix))) @cython.boundscheck(False) def dot_sigmoid_c(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, double beta, np.ndarray[double, ndim=1, mode='c'] result not None, int append_bias): cdef int ms0, ms1 ms0 = matrix.shape[0] ms1 = matrix.shape[1] dot_sigmoid(& vector[0], & matrix[0, 0], ms0, ms1, beta, & result[0], append_bias) @cython.boundscheck(False) def dot_sigmoid_c_poly(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, double beta, np.ndarray[double, ndim=1, mode='c'] result not None, int append_bias): cdef int ms0, ms1 ms0 = matrix.shape[0] ms1 = matrix.shape[1] dot_sigmoid_poly(& vector[0], & matrix[0, 0], ms0, ms1, beta, & result[0], append_bias) @cython.boundscheck(False) def dot_add_c(np.ndarray[double, ndim=1, mode='c'] vector not None, np.ndarray[double, ndim=2, mode='c'] matrix not None, np.ndarray[double, ndim=1, mode='c'] result not None, int append_bias): cdef int ms0, ms1 ms0 = matrix.shape[0] ms1 = matrix.shape[1] dot_add(& vector[0], & matrix[0, 0], ms0, ms1, & result[0], append_bias) @cython.boundscheck(False) def sigmoid_poly_c(np.ndarray[double, ndim=1, mode='c'] vector not None, double beta): cdef int vs0 vs0 = vector.shape[0] sigmoid_poly(& vector[0], vs0, beta) @cython.boundscheck(False) def sigmoid_c(np.ndarray[double, ndim=1, mode='c'] vector not None, double beta): cdef int vs0 vs0 = vector.shape[0] sigmoid(& vector[0], vs0, beta) @cython.boundscheck(False) def flip_some_bytes(int count, np.ndarray[np.uint8_t, ndim=2] arena, np.ndarray[np.int_t, ndim=1] mrange): for i in xrange(count): x = np.random.randint(0, mrange[2]) y = np.random.randint(0, mrange[3]) arena[mrange[0]+x, mrange[1]+y]=255-arena[mrange[0]+x, mrange[1]+y] cdef inline int clip(int x, int low, int high): if xhigh: return high return x @cython.boundscheck(False) def ising_model(int count, np.ndarray[np.int8_t, ndim=2] arena, np.ndarray[np.int_t, ndim=1] mrange, float beta): cdef int xmax = arena.shape[0]-2 cdef int ymax = arena.shape[1]-2 cdef int x = 0 cdef int y = 0 cdef int s = 0 cdef int de = 0 for i in xrange(count): x = clip(mrange[0]+np.random.randint(0, mrange[2]), 1, xmax) y = clip(mrange[1]+np.random.randint(0, mrange[3]), 1, ymax) s = arena[x-1, y]+arena[x+1, y]+arena[x, y-1]+arena[x, y+1] de = s*arena[x, y]-s*(-arena[x, y]) if de < 0: arena[x, y] =- arena[x, y] elif np.random.rand() 0.4: cv2.circle(image2, (y, x), 12, color=(255, 255, 255), thickness=1, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 10, color=(0, 0, 0), thickness=1, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 22, color=(255, 255, 255), thickness=1, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 20, color=(0, 0, 0), thickness=1, lineType=cv2.CV_AA) if heatmap[x, y] > 0.6: cv2.circle(image2, (y, x), 12, color=(255, 255, 255), thickness=2, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 10, color=(0, 0, 0), thickness=2, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 22, color=(255, 255, 255), thickness=2, lineType=cv2.CV_AA) cv2.circle(image2, (y, x), 20, color=(0, 0, 0), thickness=2, lineType=cv2.CV_AA) image = image.astype(float)/255 image[:, :, 0] *= heatmap image[:, :, 1] *= heatmap image[:, :, 2] *= heatmap # Construct the window self.display.place_image(0, 0, self.actual_input_prev, "Input frame") self.display.place_image(0, 1, self.predicted, "Predicted image") self.display.place_image(0, 2, self.adif[0], "State-pred.") if self.collect_error: self.display.place_image(0, 4, self.adifr[0], "Rec Difference") for layer_num in range(0, self.num_layers-1): self.display.place_image(1 + layer_num, 0, self.prop_dict['state_arrays'][layer_num].view(np.ndarray), "%dL state" % (layer_num+1)) self.display.place_image(1 + layer_num, 1, self.prop_dict['predicted_arrays'][layer_num+1].view(np.ndarray), "%dL pred. state" % (layer_num+1)) self.display.place_image(1 + layer_num, 2, self.adif[layer_num+1], "State - pred.") self.display.place_image(0 + layer_num, 4, self.prop_dict['predicted_arrays_t2'][layer_num].view(np.ndarray), "%dL pred state t+1" % (layer_num)) if self.collect_error: self.display.place_image(1 + layer_num, 4, self.adifr[layer_num+1], "Rec Difference") self.display.place_image(1 + layer_num, 3, self.prop_dict['predicted_readout_arrays'][layer_num].view(np.ndarray), "Readout %d" % layer_num) self.display.place_image(self.num_layers-1, 4, self.prop_dict['predicted_arrays_t2'][self.num_layers-1].view(np.ndarray), "%dL pred state t+1" % (self.num_layers-1)) self.display.place_image(self.num_layers, 0, self.prop_dict['state_arrays'][self.num_layers-1].view(np.ndarray), "%dL state" % self.num_layers) self.display.place_image(self.num_layers, 3, self.prop_dict['predicted_readout_arrays'][self.num_layers-1].view(np.ndarray), "Readout %d" %(self.num_layers-1)) self.display.place_image(self.num_layers, 1, max_readout_array.view(np.ndarray), "Combined readout") self.display.clear_cell(self.num_layers+1, 3) self.display.place_color_logo(self.num_layers+1, 3) self.display.place_text(self.num_layers+1, 3, voffset=50, text=self.prop_dict['name']) self.display.place_text(self.num_layers+1, 3, voffset=60, text=self.prop_dict['hash']) self.display.place_text(self.num_layers+1, 3, voffset=70, text=self.prop_dict['timestamp']) now = time.time() interval = now - self.t_prev self.t_prev = now self.display.place_text(self.num_layers+1, 3, voffset=80, text="N=%d fps=%2.2f" % (self.prop_dict['N'][0], 1 / interval)) self.display.place_text(self.num_layers+1, 3, voffset=90, text=time.ctime()) self.display.place_text(self.num_layers+1, 3, voffset=100, text="Learning rate[0] %2.5f" % self.prop_dict['learning_rates'][0][0]) self.display.place_text(self.num_layers+1, 3, voffset=110, text="Lat.conn %s, fb.conn %s " % ("yes" if self.prop_dict['context_factor_lateral'][0] > 0 else "no", "yes" if self.prop_dict['context_factor_feedback'][0] > 0 else "no")) self.display.place_text(self.num_layers+1, 3, voffset=120, text="Dream mode %s" % ("yes" if self.prop_dict['flags'][0] == 1 else "no")) self.display.place_image(self.num_layers+1, 0, image, "Superimposed") self.display.place_image(self.num_layers+1, 2, heatmap, "Heatmap") self.display.place_image(self.num_layers+1, 1, image2.astype(np.uint8), "Tracking") def developmental_check(self): # Developmental stuff for i in range(self.num_layers): if self.prop_dict["N"][0] == int(self.prop_dict["options"]["delay_each_layer_learning"])*i: self.prop_dict['learning_rates'][i][0] = float(self.prop_dict["options"]["initial_learning_rate"]) if self.prop_dict["N"][0] == int(self.prop_dict["options"]["delay_final_learning_rate"]) + \ int(self.prop_dict["options"]["delay_each_layer_learning"])*i: self.prop_dict['learning_rates'][i][0] = float(self.prop_dict["options"]["final_learning_rate"]) if "disable_lateral" not in self.prop_dict.keys() and self.prop_dict["N"][0] == int(self.prop_dict["options"]["enable_lateral_at"]): self.prop_dict['context_factor_lateral'][0] = 1.0 if "disable_feedback" not in self.prop_dict.keys() and self.prop_dict["N"][0] == int(self.prop_dict["options"]["enable_feedback_at"]): self.prop_dict['context_factor_feedback'][0] = 1.0 def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ self.developmental_check() self.readout_targets = self._get_readout_targets(channel_name="mask") self.current_frame = self.signal.get_signal(name="frame", time=0).copy() self.process_flags() if (self.do_display and self.steps % self.update_interval == 0) or self.record or self.frame_to_drop > 0: self.construct_display() # Display the window if self.do_display: cv2.imshow("PVM display", self.display.frame) if self.record: self.video_recorder.record(self.display.frame) if self.frame_to_drop>0 and self.steps==self.frame_to_drop: cv2.imwrite(self.rec_filename, self.display.frame) if self.do_display: key = cv2.waitKey(1) & 0xFF else: key = 0 if key == 27 or self.steps > self.steps_to_run: self._running = False if key == ord("a"): self.update_interval = max(1, self.update_interval-1) if key == ord("s"): self.update_interval += 1 if self.dream_experiment: self.process_dream_experiment() now = time.time() sps = self.steps / (now-self.t_start) if now - self.prev_now > 2: print ("%3.3f steps per sec" % (sps)) + "\r", sys.stdout.flush() self.prev_now = now self.steps += 1 if self.steps > self.steps_to_run: self._running = False self.signal.advance() def process_flags(self): if 'flags' in self.prop_dict.keys(): if self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_DREAM: # Dream mode self.current_frame[:] = (self.predicted*255).astype(np.uint8) elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_BLINDSPOT: # Blind spot self.current_frame = self.signal.get_signal(name="frame", time=0).copy() indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = 127 elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_GRAY: # Neutral input self.current_frame[:] = 127 elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_NOISE: # White noise self.current_frame[:] = np.random.randint(0, 255, self.current_frame.shape) elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_NOISE_SPOT: # Noise spot self.current_frame = self.signal.get_signal(name="frame", time=0).copy() indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = np.random.randint(0, 255, (indx1-indx0, indy1-indy0, 3)) elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_INV_BLINDSPOT: # Inverse blindspot self.current_frame[:] = 127 indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = self.signal.get_signal(name="frame", time=0).copy()[indx0:indx1, indy0:indy1] elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_INV_DREAM_SPOT: # Partial dream self.current_frame[:] = (self.predicted*255).astype(np.uint8) indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = self.signal.get_signal(name="frame", time=0).copy()[indx0:indx1, indy0:indy1] elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_BLINKS: # Blinks if self.prop_dict["N"][0] % 20 == 0 or self.prop_dict["N"][0] % 20 == 1: self.current_frame[:] = (self.predicted*255).astype(np.uint8) else: self.current_frame[:] = self.signal.get_signal(name="frame", time=0).copy() elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_NOISY_SIGNAL: # Noisy signal self.current_frame[:] = self.signal.get_signal(name="frame", time=0).copy() - 32+np.random.randint(0, 64, self.current_frame.shape) elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_INV_NOISE_SPOT: # Inverse spot noise self.current_frame[:] = np.random.randint(0, 255, self.current_frame.shape) indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = self.signal.get_signal(name="frame", time=0).copy()[indx0:indx1, indy0:indy1] elif self.prop_dict['flags'][0] == PVM_Create.PVM_FLAG_VAL_DREAM_SPOT: # Dream spot self.current_frame = self.signal.get_signal(name="frame", time=0).copy() indx0 = self.current_frame.shape[0]/2-self.current_frame.shape[0]/4 indx1 = indx0 + self.current_frame.shape[0]/2 indy0 = self.current_frame.shape[1]/2-self.current_frame.shape[0]/4 indy1 = indy0 + self.current_frame.shape[0]/2 self.current_frame[indx0:indx1, indy0:indy1] = (self.predicted*255).astype(np.uint8)[indx0:indx1, indy0:indy1] if self.prop_dict['flags'][PVM_Create.PVM_LEARNING_FLAG] == PVM_Create.PVM_LEARNING_FREEZE: self.freeze_learning() self.prop_dict['flags'][PVM_Create.PVM_LEARNING_FLAG] = PVM_Create.PVM_LEARNING_RESET if self.prop_dict['flags'][PVM_Create.PVM_LEARNING_FLAG] == PVM_Create.PVM_LEARNING_UNFREEZE: self.un_freeze_learning() self.prop_dict['flags'][PVM_Create.PVM_LEARNING_FLAG] = PVM_Create.PVM_LEARNING_RESET if self.prop_dict['flags'][1] == 1: self.prop_dict['flags'][1] = 0 self.dream_experiment = not self.dream_experiment logging.info("Dream experiment has been sheduled") if "finished" in self.prop_dict.keys(): if self.prop_dict['finished'][0] == PVM_Create.PVM_FLAG_VAL_TRIGGER: self._running = False self.prop_dict['finished'][0] = PVM_Create.PVM_FLAG_VAL_RESET if self.prop_dict['finished'][0] == PVM_Create.PVM_FLAG_TRIGGER_DISPLAY: self.do_display = not self.do_display if not self.do_display: cv2.destroyWindow("PVM display") cv2.destroyAllWindows() for i in range(10): cv2.waitKey(1) self.prop_dict['finished'][0] = PVM_Create.PVM_FLAG_VAL_RESET if self.prop_dict['finished'][0] == PVM_Create.PVM_FLAG_TRIGGER_RECORD: self.record = not self.record if self.record: self.do_display = True self.video_recorder.set_filename(self.prop_dict["record_filename"].tostring()) elif self.video_recorder is not None: self.video_recorder.finish() self.prop_dict['finished'][0] = 0 def process_dream_experiment(self): if self.signal.get_index() == self.prop_dict['flags'][2]: if "stage0" not in self.dream_experiment_data.keys(): self.dream_experiment_data["stage0"] = True logging.info("Dream experiment stage0 has begun") elif "stage1" not in self.dream_experiment_data.keys(): self.dream_experiment_data["stage0"] = False self.dream_experiment_data["stage1"] = True self.prop_dict['flags'][0] = 1 # begin dreaming logging.info("Dream experiment stage1 has begun") self.freeze_learning() for (i, k) in enumerate(self.prop_dict["learning_rates"]): k[0] = -0.00001 else: self.dream_experiment = False self.prop_dict['flags'][0] = 0 # end dreaming CoreUtils.save_model(self.dream_experiment_data, "dream_data.p.gz") self.dream_experiment_data = {} logging.info("Dream experiment has ended") self.un_freeze_learning() # Stage 0 is ongoing. if "stage0" in self.dream_experiment_data.keys() and self.dream_experiment_data["stage0"] is True: if "stage0_data" not in self.dream_experiment_data.keys(): self.dream_experiment_data['stage0_data'] = [] self.dream_experiment_data['stage0_data'].append((self.actual_input_prev.copy(), self.predicted.copy())) # Stage 0 is ongoing. if "stage1" in self.dream_experiment_data.keys() and self.dream_experiment_data["stage1"] is True: if "stage1_data" not in self.dream_experiment_data.keys(): self.dream_experiment_data['stage1_data'] = [] self.dream_experiment_data['stage1_data'].append(self.predicted.copy()) def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): if self.collect_error: f = open("predictive_error.txt", "w") print >>f, "%4.4f" % self.predictive_error f.close() f = open("predictive_error_array.txt", "w") for p in self.predictive_error_array: print >>f, "%4.4f" % p f.close() f = open("reconstruction_error.txt", "w") print >>f, "%4.4f" % self.reconstruction_error f.close() f = open("reconstruction_error_array.txt", "w") for p in self.reconstruction_error_array: print >>f, "%4.4f" % p f.close() if self.video_recorder is not None: self.video_recorder.finish() self.signal.finish() def freeze_learning(self): self.freeze_learning_scheduled = True def un_freeze_learning(self): self.unfreeze_learning_scheduled = True def freeze_learning_synchronised(self): """ Saves the current learning rates of the model in a temporary buffer :return: """ self.freeze_learning_scheduled = False if not self.frozen_learning: self.frozen_learning = True self.learning_rate_buffer = [] self.readout_learning_rate_buffer = [] for k in range(len(self.prop_dict['learning_rates'])): self.learning_rate_buffer.append(self.prop_dict['learning_rates'][k].copy()) self.prop_dict['learning_rates'][k][0] = 0.0 self.readout_learning_rate_buffer.append(self.prop_dict["readout_learning_rate"].copy()) self.prop_dict["readout_learning_rate"][0] = 0.0 logging.info("Freezing model learning") def un_freeze_learning_synchronised(self): """ Recovers the previously stored learning rates. :return: """ self.unfreeze_learning_scheduled = False if self.frozen_learning: for (i, k) in enumerate(self.prop_dict["learning_rates"]): k[0] = self.learning_rate_buffer[i][0] self.prop_dict["readout_learning_rate"][0] = self.readout_learning_rate_buffer[0][0] self.frozen_learning = False logging.info("Unfreezing model learning") ================================================ FILE: PVM_models/PVM_plot_error.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import argparse from PVM_framework.PVM_Storage import Storage import PVM_framework.CoreUtils as CoreUtils import numpy as np import matplotlib.pyplot as plt def runningMeanFast(x, N): return np.convolve(x, np.ones((N,))/N)[(N-1):] def plot_weight_dists(simulation_dict): all_weights_s = np.array([]) all_weights_i = np.array([]) all_weights_d = np.array([]) all_weights_e = np.array([]) all_weights_c = np.array([]) for u in simulation_dict["stage0"]: w = u['MLP_parameters']['weights'][0].copy() w_s = np.array([]) w_d = np.array([]) w_i = np.array([]) w_e = np.array([]) w_c = np.array([]) m = 0 for (i, b) in enumerate(u['signal_blocks']): n = np.prod(b[0].shape) w_s = np.hstack((w_s, w[i*4*n:i*4*n+n].flatten())) w_d = np.hstack((w_d, w[i*4*n+n:i*4*n+2*n].flatten())) w_i = np.hstack((w_i, w[i*4*n+2*n:i*4*n+3*n].flatten())) w_e = np.hstack((w_e, w[i*4*n+3*n:i*4*n+4*n].flatten())) m += 4*n if 'complex' in u.keys() and u['complex']: m = 2*np.prod(u['output_block'].shape) w = u['MLP_parameters']['weights'][1].copy() for (i, b) in enumerate(u['context_blocks']): n = np.prod(b[0].shape) w_c = np.hstack((w_c, w[m+i*n:m+i*n+n].flatten())) else: for (i, b) in enumerate(u['context_blocks']): n = np.prod(b[0].shape) w_c = np.hstack((w_c, w[m+i*n:m+i*n+n].flatten())) all_weights_s = np.hstack((all_weights_s, w_s)) all_weights_i = np.hstack((all_weights_i, w_i)) all_weights_d = np.hstack((all_weights_d, w_d)) all_weights_e = np.hstack((all_weights_e, w_e)) all_weights_c = np.hstack((all_weights_c, w_c)) plt.figure(figsize=(14, 5)) plt.subplot(121) all_weights = [all_weights_s, all_weights_i, all_weights_d, all_weights_e, all_weights_c] plt.hist(all_weights, 15, normed=0, color=['red', 'yellow', 'green', 'gray', 'blue'], label=['Signal', 'Integral', 'Derivative', 'Error', 'Context']) plt.grid(True) plt.title("Weight distribution " + simulation_dict['name']) plt.legend(prop={'size': 9}) plt.subplot(122) plt.hist(map(lambda x: np.abs(x), all_weights), 15, normed=0, color=['red', 'yellow', 'green', 'gray', 'blue'], label=['Signal', 'Integral', 'Derivative', 'Error', 'Context']) plt.grid(True) plt.title("Weight distribution (abs) " + simulation_dict['name']) plt.legend(prop={'size': 9}) pdf_file = "PVM_%s_%s_weight_dist.pdf" % (simulation_dict['name'], simulation_dict['hash']) plt.savefig(pdf_file) return pdf_file def plot_model(filename, remote, compare, display): ts = Storage() if remote != "": filename = ts.get(remote) if compare != "": filename1 = ts.get(compare) simulation_dict1 = CoreUtils.load_model(filename1) simulation_dict = CoreUtils.load_model(filename) num_layers = len(simulation_dict["state_arrays"]) plt.figure(figsize=(7, 5)) colors = ['r', 'g', 'b', 'k', 'c', 'm', 'y'] ids = np.where(simulation_dict['error_log'][0, :] > 0)[0] window = (ids.shape[0])/200 small_window = (ids.shape[0])/40 for r in range(num_layers): # plt.suptitle("MSE %s" % (simulation_dict['name'])) plt.plot(simulation_dict['error_log'][0, ids][:-small_window], runningMeanFast(simulation_dict['error_log'][r+1, ids], small_window)[:-small_window], lw=1, c=colors[r % 7], label="%s l. %d" % (simulation_dict['name'], r)) if compare: for r in range(num_layers): plt.plot(simulation_dict1['error_log'][0, ids][:-small_window], runningMeanFast(simulation_dict1['error_log'][r+1, ids], small_window)[:-small_window], linestyle="--", lw=1, c=colors[r % 7], label="%s l. %d" % (simulation_dict1['name'], r)) plt.xlabel("Training time") plt.ylabel("MSE (averaged in %d step bins)" % (1000*small_window)) plt.title("Learning curve (MSE) - individual layers") plt.grid(True) plt.legend(prop={'size': 9}) pdf_file = "PVM_%s_%s.pdf" % (simulation_dict['name'], simulation_dict['hash']) if compare != "": pdf_file = "PVM_%s_%s_comp_%s.pdf" % (simulation_dict['name'], simulation_dict['hash'], simulation_dict1['name']) plt.savefig(pdf_file) to_folder = "DARPA/Simulations/%s_%s_%s/" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) ts.put(from_path=pdf_file, to_folder=to_folder, overwrite=True) plt.figure(figsize=(7, 5)) colors = ['r', 'g', 'b', 'k', 'c', 'm', 'y'] ids = np.where(simulation_dict['error_log'][0, :] > 0)[0] summed = np.zeros_like(simulation_dict['error_log'][1, ids]) for r in range(num_layers): summed += simulation_dict['error_log'][r+1, ids] plt.plot(simulation_dict['error_log'][0, ids][:-small_window], runningMeanFast(summed, small_window)[:-small_window], lw=1, c='r', label="MSE All layers, model %s" % simulation_dict['name']) if compare != "": summed = np.zeros_like(simulation_dict1['error_log'][1, ids]) for r in range(num_layers): summed += simulation_dict1['error_log'][r+1, ids] plt.plot(simulation_dict1['error_log'][0, ids][:-small_window], runningMeanFast(summed, small_window)[:-small_window], lw=1, c='b', label="MSE All layers, model %s" % simulation_dict1['name']) plt.xlabel("Training time") plt.ylabel("MSE (averaged in %d step bins)" % (1000*small_window)) plt.title("Learning curve (MSE) - whole system") plt.grid(True) plt.legend(prop={'size': 9}) pdf_file = "PVM_%s_%s_summed.pdf" % (simulation_dict['name'], simulation_dict['hash']) if compare != "": pdf_file = "PVM_%s_%s_summed_comp_%s.pdf" % (simulation_dict['name'], simulation_dict['hash'], simulation_dict1['name']) plt.savefig(pdf_file) to_folder = "DARPA/Simulations/%s_%s_%s/" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) ts.put(from_path=pdf_file, to_folder=to_folder, overwrite=True) # pdf_file = plot_weight_dists(simulation_dict) # ts.put(from_path=pdf_file, to_folder=to_folder, overwrite=True) # if compare != "": # pdf_file = plot_weight_dists(simulation_dict1) # ts.put(from_path=pdf_file, to_folder=to_folder, overwrite=True) if display: plt.show() if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="File to load", type=str, default="") parser.add_argument("-r", "--remote", help="Download and run a remote simulation", type=str, default="") parser.add_argument("-c", "--compare", help="Compare with this file", type=str, default="") parser.add_argument("-D", "--display", help="Pull up display window", action="store_true") args = parser.parse_args() plot_model(filename=args.file, remote=args.remote, compare=args.compare, display=args.display) ================================================ FILE: PVM_models/PVM_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import os import logging import argparse import subprocess import re import string import json from PVM_models.PVM_Manager import Manager import PVM_framework.CoreUtils as CoreUtils import PVM_framework.PVM_Storage as PVM_Storage import PVM_framework.PVM_Create as PVM_Create import PVM_framework.PVM_datasets as PVM_datasets import PVM_framework.PVM_SignalProvider as PVM_SignalProvider import PVM_framework.PVM_options as PVM_options import PVM_framework.PVM_display_helper as PVM_display_helper from boto.utils import get_instance_metadata def cleanup(s): ansi_escape = re.compile(r'\x1b[^m]*m') s = ansi_escape.sub('', s) s = s.strip() s = filter(lambda x: x in string.printable, s) return s def run_model(evaluate=False, filename="", cores="", name="", description="", remote="", display=False, dataset="", meta={}, options_given={}, storage=None, port="9000", checkpoint=True, upgrade_only=False): """ In this demo a future/predictive encoder is being instantiated to predict a camera image based on two previous frames. The system is built into a three layer hierarchy in which each next layer is predicting the hidden activations of the lower one. In addition the errors from each later are being backpropagated down to the previous layer. In addition to that, errors generated at context blocka are also being backpropagated to the originating unit. Consequently the error and signals flows in both directions through the entire system. """ if options_given == {} and filename == "" and remote == "": logging.error("No options were given, don't know what to run! Try running with -h option.") exit() options = PVM_options.parse_options(options_given) if remote != "": filename = storage.get(remote) logging.info("Loaded a remote simulation dict %s" % remote) logging.info("Following options were given: %s" % json.dumps(options, sort_keys=True, indent=4)) if os.path.isfile(filename): simulation_dict = CoreUtils.load_model(filename) if "options" in simulation_dict: options = PVM_options.parse_options(options_given, options_in_the_dict=simulation_dict['options']) else: options = PVM_options.parse_options(options_given) logging.info("Loaded the dictionary") if cores is not "": simulation_dict['num_proc'] = int(cores) else: simulation_dict['num_proc'] = min(2*mp.cpu_count()/3, simulation_dict["stage0_size"]/2) PVM_Create.upgrade(simulation_dict) PVM_Create.upgrade_dictionary_to_ver1_0(simulation_dict) logging.info("Running on %d cpu's" % simulation_dict['num_proc']) else: options = PVM_options.parse_options(options) simulation_dict = PVM_Create.generate_dict_options(name=name, description=description, options=options ) if cores is not "": simulation_dict['num_proc'] = int(cores) else: if options["model_type"] != "tiny": simulation_dict['num_proc'] = 2*mp.cpu_count()/3 else: simulation_dict['num_proc'] = 1 logging.info("Generated the dictionary") logging.info("Full set of options: %s" % json.dumps(options, sort_keys=True, indent=4)) if options["new_name"] != "": simulation_dict['name'] = options["new_name"] options["new_name"] = "" if "disable_lateral" in options.keys() and options["disable_lateral"] == "1": simulation_dict["disable_lateral"] = True if "disable_feedback" in options.keys() and options["disable_feedback"] == "1": simulation_dict["disable_feedback"] = True if dataset == "": dataset = options["dataset"] else: options["dataset"] = dataset PVM_set = PVM_datasets.PVMDataset(dataset, storage=storage) PVM_Create.apply_options(simulation_dict, options) if upgrade_only: CoreUtils.save_model(simulation_dict, filename) to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) from_path = filename storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) return if options['supervised'] == "1": logging.info("Running in the supervised mode") for (i, k) in enumerate(simulation_dict['learning_rates']): k[0] = 0 logging.info("Setting learning rate %d to zero") simulation_dict['readout_learning_rate'][0] = float(options["supervised_rate"]) logging.info("Setting additional_learning_rate to %f" % simulation_dict['readout_learning_rate'][0]) if not evaluate: status_file = "/tmp/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) f = open(status_file, "w") branch = subprocess.Popen('git rev-parse --abbrev-ref HEAD', shell=True, stdout=subprocess.PIPE).stdout.read() f.write("BRANCH=%s\n" % cleanup(branch)) f.write("TIMESTAMP=%s\n" % simulation_dict['timestamp']) f.write("NAME=%s\n" % simulation_dict['name']) f.write("HASH=%s\n" % simulation_dict['hash']) f.write("DATASET=%s\n" % dataset) f.write("OPTIONS=%s\n" % json.dumps(options)) f.close() remove_artifact_files = True if meta == {}: logging.info("Not running on an Amazon EC2 instance, apparently") logging.info("Not running on an Amazon EC2 instance, apparently: So, not automatically removing downloaded artifact files.") remove_artifact_files = False elif options['supervised'] != '1': host = meta['public-ipv4'] logging.info("Running on amazon instance %s. Adding active job" % host) storage.put(from_path=status_file, to_folder='DARPA/active_jobs/', overwrite=True) # Train signal = PVM_SignalProvider.SimpleSignalProvider(files=PVM_set.training, storage=storage, frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]), heatmap_resolution=simulation_dict['readout_arrays'][0].shape[:2][::-1], channel="default", remove_files=remove_artifact_files, reverse=(int(options['reverse']) > 0)) manager = Manager(simulation_dict, int(options['steps']), signal_provider=signal, record=False, video_recorder=PVM_display_helper.VideoRecorder(rec_filename="PVM_recording.avi"), do_display=display, checkpoint=checkpoint, checkpoint_storage=storage, dataset_name=dataset) CoreUtils.run_model(simulation_dict, manager, port=int(port)) if filename != "" and options['supervised'] != "1": CoreUtils.save_model(simulation_dict, filename) to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) from_path = filename storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) if remove_artifact_files: os.remove(from_path) elif options['supervised'] == "1": CoreUtils.save_model(simulation_dict, "PVM_state_supervised_%s_%d_%d_%f.p.gz" % (dataset, simulation_dict['N'][0], int(options['steps']), float(options['supervised_rate']))) to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) from_path = "./PVM_state_supervised_%s_%d_%d_%f.p.gz" % (dataset, simulation_dict['N'][0], int(options['steps']), float(options['supervised_rate'])) storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) if remove_artifact_files: os.remove(from_path) else: CoreUtils.save_model(simulation_dict, "PVM_state_final.p.gz") to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash']) from_path = "./PVM_state_final.p.gz" storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) if remove_artifact_files: os.remove(from_path) else: print "Evaluating the system" logging.info("Evaluating the system") to_folder = "PVM_models/%s_%s_%s/eval_%09d/" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'], simulation_dict['N']) # Evaluate signal = PVM_SignalProvider.SimpleSignalProvider(files=PVM_set.testing, storage=storage, frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]), heatmap_resolution=simulation_dict['readout_array_float00'].shape[:2][::-1], channel="default", reverse=(int(options['reverse']) > 0)) name = "PVM_train_eval_%s_%09d_test_combined.avi" % (simulation_dict['hash'], simulation_dict['N']) manager = Manager(simulation_dict, steps_to_run=-1, signal_provider=signal, record=True, video_recorder=PVM_display_helper.VideoRecorder(rec_filename=name), do_display=display, evaluate=True, collect_error=True) manager.freeze_learning() CoreUtils.run_model(simulation_dict, manager, port=int(port)) from_path = name storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) os.remove(name) logging.info("Finished on test files") # Individual files for (i, test) in enumerate(PVM_set.all): print "Running on %s" % test logging.info("Running on %s" % test) name = "PVM_eval_%s_%09d_%01d_%s.avi" % (simulation_dict['hash'], simulation_dict['N'], i, test[1]) signal = PVM_SignalProvider.SimpleSignalProvider(files=[test], storage=storage, frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]), heatmap_resolution=simulation_dict['readout_array_float00'].shape[:2][::-1], channel="default") manager = Manager(simulation_dict, steps_to_run=-1, signal_provider=signal, record=True, video_recorder=PVM_display_helper.VideoRecorder(rec_filename=name), do_display=display, evaluate=True) CoreUtils.run_model(simulation_dict, manager, port=int(port)) from_path = name storage.put(from_path=from_path, to_folder=to_folder, overwrite=True) os.remove(name) logging.info("Finished on %s" % test) if __name__ == '__main__': logging.basicConfig(filename="PVM.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(thread)d PVM_run : %(message)s ') logging.info("###################################################################") logging.info(" STARTING NEW RUN ") logging.info("###################################################################") parser = argparse.ArgumentParser(description=PVM_options.get_option_help(), formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-e", "--evaluate", help="Evaluate the trained system on a set of test movies", action="store_true") parser.add_argument("-B", "--backup", help="Store a simulation snapshot every 100.000 steps", action="store_true") parser.add_argument("-q", "--quiet", help="Print minimal logs to stdout", action="store_true") parser.add_argument("-D", "--display", help="Pull up display window", action="store_true") parser.add_argument("-u", "--upgrade_only", help="Load and upgrade the dictionary but don't run", action="store_true") parser.add_argument("-f", "--file", help="File to load", type=str, default="") parser.add_argument("-c", "--cores", help="Number of cores to use", type=str, default="") parser.add_argument("-d", "--description", help="Short description of the simulation", type=str, default="") parser.add_argument("-r", "--remote", help="Download and run a remote simulation", type=str, default="") parser.add_argument("-n", "--name", help="Short name of the simulation", type=str, default="PVM_simulation") parser.add_argument("-s", "--set", help="Name of the dataset", type=str, default="") parser.add_argument("-S", "--spec", help="Specification file name (file in .json format)", type=str, default="") parser.add_argument("-p", "--port", help="Debug console port. Access debug console by typing \"nc localhost _port_\"", type=str, default="9000") parser.add_argument('-O', '--options', type=json.loads, help="Option dictionary (as described above) given in json form '{\"key1\": \"value1\"}\'.", default='{}') Storage = PVM_Storage.Storage() try: meta = get_instance_metadata(timeout=2, num_retries=2) except: meta = {} args = parser.parse_args() have_display = args.display if not args.quiet: logging.getLogger().addHandler(logging.StreamHandler()) try: from Tkinter import Tk Tk() except: have_display = False print "No display detected, turning off visualizations." using_cmdline_options = len(args.options) > 0 using_json_options = len(args.spec) > 0 assert not (using_cmdline_options and using_json_options), "We don't support using both command line and json spec options at the same time. (Suggestion: put all options in the json spec)" options = args.options if args.spec != "": options = json.load(open(args.spec, "r")) run_model(evaluate=args.evaluate, filename=args.file, cores=args.cores, description=args.description, display=have_display, name=args.name, remote=args.remote, dataset=args.set, meta=meta, options_given=options, storage=Storage, port=args.port, checkpoint=args.backup, upgrade_only=args.upgrade_only ) ================================================ FILE: PVM_models/PVM_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import matplotlib matplotlib.use('Agg') import PVM_framework.CoreUtils as CoreUtils import PVM_framework.PVM_Create as PVM_Create import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import logging import cv2 from PVM_tools.abstract_tracker import GenericVisionTracker from PVM_tools.bounding_region import BoundingRegion import numpy as np class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True def start(self): """ This will be called right before the simulation starts """ pass def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ pass def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ pass def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass class PVMVisionTracker(GenericVisionTracker): """ This class exposes the null vision tracker which just always returns its priming bounding box """ def __init__(self, filename="", remote_filename="", cores="4", storage=None, steps_per_frame=1): """ Initialize the tracker """ self.name = 'PVMtracker' if filename == "": filename = storage.get(remote_filename) self.prop_dict = CoreUtils.load_model(filename) logging.info("Loaded the dictionary %s", filename) PVM_Create.upgrade_dictionary_to_ver1_0(self.prop_dict) self.prop_dict['num_proc'] = int(cores) for k in range(len(self.prop_dict['learning_rates'])): self.prop_dict['learning_rates'][k][0] = 0.0 logging.info("Setting learning rate in layer %d to zero" % k) self.prop_dict["readout_learning_rate"][0] = 0.0 logging.info("Setting readout learning rate to zero") self.manager = Manager(self.prop_dict, 1000) self.executor = CoreUtils.ModelExecution(prop_dict=self.prop_dict, manager=self.manager, port=9100) self.executor.start(blocking=False) self.threshold = 32 self.image_size = self.prop_dict['input_array'].shape[:2][::-1] self.readout_heatmap = np.zeros(self.image_size, dtype=np.float) self.step_per_frame = steps_per_frame def reset(self): """ Reset the tracker :return: """ self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ if not self._primed: logging.info("Priming the PVM tracker") logging.info("Setting up the tracker threshold to %d" % self.threshold) self._primed = True def _track(self, im): """ Track on given image, rseturn a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again current_frame = cv2.resize(im, dsize=self.image_size) self.prop_dict['input_array'][:] = current_frame self.prop_dict['input_array_float'][:] = current_frame.astype(np.float)/255 for i in range(self.step_per_frame): self.executor.step() norm = 1.0 / len(self.prop_dict['predicted_readout_arrays']) self.readout_heatmap[:] = 0 for k in self.prop_dict['predicted_readout_arrays']: self.readout_heatmap[:] += cv2.resize(k.view(np.ndarray), dsize=self.image_size) * norm am = np.unravel_index(np.argmax(self.readout_heatmap), self.readout_heatmap.shape) if len(am) == 3: for d in range(3): if am[2] != d: self.readout_heatmap[:, :, d] = 0 self.heatmap = self.readout_heatmap.view(np.ndarray) if len(self.heatmap.shape) == 3: self.heatmap = np.max(self.heatmap, axis=2) self.heatmap = cv2.resize(self.heatmap, dsize=(im.shape[1], im.shape[0]), interpolation=cv2.INTER_CUBIC) self.heatmap = (self.heatmap * 255).astype(np.uint8) if np.max(self.heatmap) > (np.median(self.heatmap)+self.threshold): threshold=(np.max(self.heatmap)-np.median(self.heatmap))*0.5+np.median(self.heatmap) ret, thresh = cv2.threshold(self.heatmap, threshold, 255, 0) contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) if len(contours) > 0: max_cnt=None for cnt in contours: pt = np.unravel_index(np.argmax(self.heatmap), self.heatmap.shape) if cv2.pointPolygonTest(cnt, (pt[1], pt[0]), False)>=0: max_cnt = cnt break if max_cnt is None: self.bbox = BoundingRegion() else: x, y, w, h = cv2.boundingRect(max_cnt) if w*h > 25: self.bbox = BoundingRegion(image_shape=im.shape, box=[x, y, w, h]) self.bbox.scale(1.1) else: self.bbox = BoundingRegion() else: self.bbox = BoundingRegion() else: self.bbox = BoundingRegion() self._primed = False return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.heatmap() def finish(self): self.manager._running = False self.executor.step() self.executor.finish() ================================================ FILE: PVM_models/PVM_unit_2step_residual_v1.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.PVM_Create as PVM_Create import PVM_framework.MLP as MLP import copy class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): """ Unit architecture: [internal pred] + [residual prediction] / \ \ / / \ [OUTPUT] <- compressed residual error / \ / \ [CONTEXT] [PAST SIGNAL] Features used by the unit to predict each block: - previous value of the block - error in previous prediction - temporal derivative of the block - integral of the block - ? """ # signal_block, delta_block, prediction_t+1, prediction_t+2 SIGNAL_BLOCK_CONTENTS = set(["signal_block: Raw signal block", "delta_block: Delta extracted from associator", "prediction_tp1: Predicted block at t+1", "prediction_tp2: Predicted block at t+2"]) UNSUPERVISED_SIGNAL_INPUTS = set(["block: Raw signal block t-1", "past_block: Raw signal block t-2"]) UNSUPERVISED_CONTEXT_INPUTS = set(["block: Raw feedback or context input Block", "delta: Delta of the context inputs [currently NOT computed or used]", "factor: Factor that scales this feedback or context input"]) # Each block: teaching_signal, delta_block, readout_block SUPERVISED_TASK_OUTPUTS = set(["teaching_block: Target heatmap", "delta: Prediction error of heatmap", "readout_block: Prediction of heatmap"]) @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 3 # because there is execute0, 1 and 2 implemented def open_views_into_shmem(self, shared_memory, key, num_items): return map(lambda x: tuple([x[i].view(np.ndarray) for i in range(num_items)]), shared_memory[key]) def __init__(self, parameters): # Mapping SharedArray items through .view(np.ndarray) improved the access time self.signal_blocks = self.open_views_into_shmem(parameters, "signal_blocks", len(ExecutionUnit.SIGNAL_BLOCK_CONTENTS)) self.readout_blocks = self.open_views_into_shmem(parameters, "readout_blocks", len(ExecutionUnit.SUPERVISED_TASK_OUTPUTS)) self.context_blocks = self.open_views_into_shmem(parameters, "context_blocks", len(ExecutionUnit.UNSUPERVISED_CONTEXT_INPUTS)) self.internal_buffers = self.open_views_into_shmem(parameters, "internal_buffers", 3) self.output_block = parameters['output_block'].view(np.ndarray) # Will appear in other places as either input or context self.output_min = parameters['output_min'] self.output_max = parameters['output_max'] # The perceptron self.MLP_internal_prediction = MLP.MLP(parameters["Primary_Predictor_params"]) self.MLP_residual_prediction = MLP.MLP(parameters["Residual_Predictor_params"]) self.MLP_readout = MLP.MLP(parameters["Readout_Predictor_params"]) # This is a "task" supervised MLP (e.g., tracker) self.primary_learning_rate = parameters['primary_learning_rate'] self.readout_learning_rate = parameters['readout_learning_rate'] self.layers_internal_prediction = parameters["Primary_Predictor_params"]['layers'] self.layers_residual_prediction = parameters["Residual_Predictor_params"]['layers'] self.layers_readout = parameters["Readout_Predictor_params"]['layers'] # These are the "task" supervised MLP layers self.tau = parameters['tau'] # Tau is the integration constant for the signal integral # Input buffers self.ninputs = 0 self.npredictions = 0 self.npinputs = 0 self.ncontexts = 0 for (block, delta, pred_block1, pred_block2) in self.signal_blocks: self.ninputs += len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS) * np.prod(block.shape) self.npredictions += np.prod(block.shape) for (teaching_block, delta, readout_block) in self.readout_blocks: self.npinputs += np.prod(teaching_block.shape) self.inputs_t = np.zeros((self.ninputs,)) self.actual_signal_t = np.zeros((self.npredictions,)) self.readout_training_signal = np.zeros((self.npinputs,)) self.inputs_t_1 = np.zeros((self.ninputs,)) self.actual_signal_t_1 = np.zeros((self.npredictions,)) self.pinputs_t_1 = np.zeros((self.npinputs,)) # Context buffers for (block, delta, factor) in self.context_blocks: self.ncontexts += np.prod(block.shape) self.contexts_t_1 = np.zeros((self.ncontexts,)) self.output_layer = len(self.layers_residual_prediction)-2 # Becuse the "output" of this Unit comes from its hidden layer self.output_length = np.prod(self.output_block.shape) # Buffer for storing activations self.activations_buffer = [] # additional flags if 'complex' in parameters.keys() and parameters['complex']: self.complex = True else: self.complex = False self.push_activation() @staticmethod def upgrade_to_ver_1(parameters): parameters['internal_buffers'] = [] parameters['output_min'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['output_max'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: block01 = SharedArray.SharedNumpyArray_like(block) block02 = SharedArray.SharedNumpyArray_like(block) block03 = SharedArray.SharedNumpyArray_like(block) parameters['internal_buffers'].append((block01, block02, block03)) @staticmethod def generate_missing_parameters(parameters, options): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate. When complex_unit is False, a standard 3-layer MLP is used. When complex_unit is True, an MLP with additional hidden layers is used. There needs to be no return value, the method leaves a side effect by modifying the perameters dict. :param parameters: parameter dictionary :type parameters: dict """ complex_unit = options['unit_type'] == "complex" polynomial = options['polynomial'] == '1' autoencoder = options['autoencoder'] == '1' nhidden = np.prod(parameters['output_block'].shape) parameters['output_min'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['output_max'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) ninputs = 0 noutputs = 0 ncontext = 0 # Any additional memory buffers needed in the operation of the unit parameters['internal_buffers'] = [] for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: block01 = SharedArray.SharedNumpyArray_like(block) block02 = SharedArray.SharedNumpyArray_like(block) block03 = SharedArray.SharedNumpyArray_like(block) parameters['internal_buffers'].append((block01, block02, block03)) for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: ninputs += np.prod(block.shape) * len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS) for (block, delta, factor) in parameters['context_blocks']: ncontext += np.prod(block.shape) for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: noutputs += 2 * np.prod(block.shape) # to predict two steps into the future nadditional = 0 for (block, delta, pblock) in parameters['readout_blocks']: nadditional += np.prod(block.shape) parameters["Primary_Predictor_params"] = {} parameters["Residual_Predictor_params"] = {} parameters["Readout_Predictor_params"] = {} if complex_unit: # 4 layer perceptron parameters["Primary_Predictor_params"]['layers'] = MLP.get_layers([ncontext+1, 3*nhidden+1, 2*nhidden+1, noutputs+1]) parameters["Residual_Predictor_params"]['layers'] = MLP.get_layers([ninputs+1, 2*nhidden+1, nhidden+1, noutputs+1]) parameters["complex"] = True else: # 3 layer perceptron Simple MLP Unit (not complex unit) parameters["Primary_Predictor_params"]['layers'] = MLP.get_layers([ncontext+1, 2*nhidden+1, noutputs+1]) parameters["Residual_Predictor_params"]['layers'] = MLP.get_layers([ninputs+2*nhidden+1, nhidden+1, noutputs+1]) parameters["Primary_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Primary_Predictor_params"]['beta'][0] = 1.0 parameters["Primary_Predictor_params"]['learning_rate'] = parameters['primary_learning_rate'] parameters["Primary_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Primary_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Primary_Predictor_params"]['weights'] = MLP.get_weights(parameters["Primary_Predictor_params"]['layers']) parameters["Residual_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Residual_Predictor_params"]['beta'][0] = 1.0 parameters["Residual_Predictor_params"]['learning_rate'] = parameters['primary_learning_rate'] parameters["Residual_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Residual_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Residual_Predictor_params"]['weights'] = MLP.get_weights(parameters["Residual_Predictor_params"]['layers']) parameters["Readout_Predictor_params"]['layers'] = MLP.get_layers([nhidden+1, 2*nhidden+1, nadditional+1]) parameters["Readout_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['beta'][0] = 1.0 parameters["Readout_Predictor_params"]['learning_rate'] = parameters['readout_learning_rate'] parameters["Readout_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Readout_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['weights'] = MLP.get_weights(parameters["Readout_Predictor_params"]['layers']) parameters["Primary_Predictor_params"]['polynomial'] = polynomial parameters["Residual_Predictor_params"]['polynomial'] = polynomial parameters["Readout_Predictor_params"]['polynomial'] = polynomial parameters['autoencoder'] = autoencoder def min_max_normalize(self, a, min_a, max_a): """ Simple min-max normalization :param a: array to be normalized :param min_a: recorded min values :param max_a: recorded max values :return: """ if self.primary_learning_rate[0] != 0: min_a[:] = np.minimum(a, min_a) + 0.000001 max_a[:] = np.maximum(a, max_a) - 0.000001 return np.divide(a-min_a, max_a-min_a) def execute0(self): """ Make predictions based on available information and publish them. """ # Prepare the input vector self.layers_internal_prediction[0]['activation'][:-1] = self.contexts_t_1 # Make internal prediction self.MLP_internal_prediction.mlp_forward() # Make residual prediction self.layers_residual_prediction[0]['activation'][:-1] = np.concatenate((self.inputs_t, self.layers_internal_prediction[1]["activation"][:-1])) self.MLP_residual_prediction.mlp_forward() # Forward internal representations to the readout predictor self.layers_readout[0]['activation'][:-1] = self.layers_residual_prediction[self.output_layer]['activation'][:-1] # Make readout prediction self.MLP_readout.mlp_forward() # Update the output self.output_block[:] = self.min_max_normalize((self.layers_residual_prediction[self.output_layer]['activation'][:self.output_length]).reshape(self.output_block.shape), self.output_min, self.output_max) # Publish predictions (mainly for debugging and inspection) i = 0 s = len(self.signal_blocks) for (idx, (block, delta, pred_block1, pred_block2)) in enumerate(self.signal_blocks): l = np.prod(pred_block1.shape) pred_block1[:] = np.clip((self.layers_internal_prediction[-1]['activation'][i:i+l] + 2 * (self.layers_residual_prediction[-1]['activation'][i:i+l]-0.5)).reshape(pred_block1.shape), 0, 1) pred_block_local = self.internal_buffers[idx][0] pred_block_local[:] = np.clip((self.layers_internal_prediction[-1]['activation'][i:i+l] + 2*(self.layers_residual_prediction[-1]['activation'][i:i+l]-0.5)).reshape(pred_block_local.shape), 0, 1) pred_block2[:] = np.clip((self.layers_internal_prediction[-1]['activation'][s*l+i:s*l+i+l] + 2 * (self.layers_residual_prediction[-1]['activation'][s*l+i:s*l+i+l]-0.5)).reshape(pred_block1.shape), 0, 1) i += l i = 0 for (block, delta, readout_block) in self.readout_blocks: l = np.prod(readout_block.shape) readout_block[:] = (self.layers_readout[-1]['activation'][i:i+l]).reshape(readout_block.shape) i += l # Since predicting two steps into the future, learning can only be performed on previous # activations, hence current activations are pushed into the FIFO if self.primary_learning_rate[0] != 0: self.push_activation() def execute1(self): """ Collect new signals, perform association, prepare for additional learning """ # Collect the current signal i = 0 j = 0 for (idx, (block, delta, pred_block1, pred_block2)) in enumerate(self.signal_blocks): l = np.prod(block.shape) # Calculate signal features past_block = self.internal_buffers[idx][1] self.inputs_t[i:i + l] = block.flatten() self.inputs_t[i + l:i + 2 * l] = past_block.flatten() past_block[:] = block i += 2*l # Collect the current signal as supervising signal self.actual_signal_t[j:j+l] = block.flatten() j += l # Predictive associative training if self.primary_learning_rate[0] != 0: # pop the previous activaitons from the FIFO self.pop_activation() # Create the expected output block training_signal = np.concatenate((self.actual_signal_t_1, self.actual_signal_t)) # Calculate the prediction error error_primary = training_signal - self.layers_internal_prediction[-1]['activation'][:np.prod(training_signal.shape)] error_residual = (error_primary + 1) * 0.5 - self.layers_residual_prediction[-1]['activation'][:np.prod(training_signal.shape)] self.actual_signal_t_1[:] = self.actual_signal_t # Make the association self.MLP_internal_prediction.train2(error=error_primary) self.MLP_residual_prediction.train2(error=error_residual) # Readout training if self.readout_learning_rate[0] != 0: # Collect the curent readout training signal i = 0 for (block, delta, pblock) in self.readout_blocks: l = np.prod(block.shape) self.readout_training_signal[i:i + l] = block.flatten() i += l error_readout = self.readout_training_signal - self.layers_readout[-1]['activation'][:-1] self.MLP_readout.train2(error=error_readout) def execute2(self): """ Gather the context signal in preparation for the next step. Some context blocks get their information from units lateral to this one, others get their information from units above this one. The "factor" is either the context_factor_lateral or context_factor_feedback """ # Prepare information for the next step # collect the new context i = 0 for (block, delta, factor) in self.context_blocks: l = np.prod(block.shape) self.contexts_t_1[i:i+l] = factor[0]*block.flatten() i += l def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass def push_activation(self): """ Save the current network activations to a FIFO queue :return: """ layers_copy = copy.deepcopy(self.layers_internal_prediction) layers_res_copy = copy.deepcopy(self.layers_residual_prediction) self.activations_buffer.append((layers_copy, layers_res_copy)) def pop_activation(self): """ Pop the first available activations from the FIFO :return: """ (layers, layers_res) = self.activations_buffer.pop(0) for layer in range(len(layers)): self.layers_internal_prediction[layer]['activation'][:] = layers[layer]['activation'] self.layers_internal_prediction[layer]['error'][:] = layers[layer]['error'] self.layers_internal_prediction[layer]['delta'][:] = layers[layer]['delta'] for layer in range(len(layers_res)): self.layers_residual_prediction[layer]['activation'][:] = layers_res[layer]['activation'] self.layers_residual_prediction[layer]['error'][:] = layers_res[layer]['error'] self.layers_residual_prediction[layer]['delta'][:] = layers_res[layer]['delta'] ================================================ FILE: PVM_models/PVM_unit_test.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): """ Does everything the PVM unit does, but at the end just copies its previous input to the output. Used for testing and debugging. """ SIGNAL_BLOCK_CONTENTS = set(["block: Raw signal block", "delta: Delta extracted from associator", "pred_block: Predicted block, output", "past_block: Past block, input", "dblock: Derivative of raw signal block", "iblock: Integral of raw signal block", "pred_block_local: Clean local copy of predicted block"]) UNSUPERVISED_SIGNAL_INPUTS = set(["block: Raw signal block", "dblock: Derivative of raw signal block", "iblock: Integral of raw signal block", "e_block: Error in prediction"]) UNSUPERVISED_CONTEXT_INPUTS = set(["block: Raw feedback or context input Block", "delta: Delta of the context inputs [currently NOT computed or used]", "factor: Factor that scales this feedback or context input"]) SUPERVISED_TASK_OUTPUTS = set(["block: Target heatmap", "delta: Prediction error of heatmap", "pblock: Prediction of heatmap", "pdblock: Prediction of derivative of heatmap"]) @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 3 # because there is execute0, 1 and 2 implemented def open_views_into_shmem(self, shared_memory, key, num_items): return map(lambda x: tuple([x[i].view(np.ndarray) for i in range(num_items)]), shared_memory[key]) def __init__(self, parameters): # Mapping SharedArray items through .view(np.ndarray) improved the access time self.signal_blocks = self.open_views_into_shmem(parameters, "signal_blocks", len(ExecutionUnit.SIGNAL_BLOCK_CONTENTS)) self.predicted_blocks = self.open_views_into_shmem(parameters, "predicted_blocks", len(ExecutionUnit.SUPERVISED_TASK_OUTPUTS)) self.context_blocks = self.open_views_into_shmem(parameters, "context_blocks", len(ExecutionUnit.UNSUPERVISED_CONTEXT_INPUTS)) self.output_block = parameters['output_block'].view(np.ndarray) # Will appear in other places as either input or context # The perceptron self.MLP = MLP.MLP(parameters["MLP_parameters"]) self.MLP_1 = MLP.MLP(parameters["MLP_parameters_additional"]) # This is a "task" supervised MLP (e.g., tracker) self.learning_rate = parameters["MLP_parameters"]['learning_rate'] self.learning_rate_1 = parameters["MLP_parameters_additional"]['learning_rate'] self.layers = parameters["MLP_parameters"]['layers'] self.layers_1 = parameters["MLP_parameters_additional"]['layers'] # These are the "task" supervised MLP layers self.tau = parameters['tau'] # Tau is the integration constant for the signal integral # Input buffers self.ninputs = 0 self.npredictions = 0 self.npinputs = 0 self.ncontexts = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in self.signal_blocks: self.ninputs += len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS) * np.prod(block.shape) self.npredictions += np.prod(block.shape) for (block, delta, pblock, pdblock) in self.predicted_blocks: self.npinputs += np.prod(block.shape) self.inputs_t = np.zeros((self.ninputs,)) self.predictions_t = np.zeros((self.npredictions,)) self.pinputs_t = np.zeros((self.npinputs,)) # Context buffers for (block, delta, factor) in self.context_blocks: self.ncontexts += np.prod(block.shape) self.contexts_t_1 = np.zeros((self.ncontexts,)) # Buffer for storing the averaged out deltas self.output_layer = len(self.layers)-2 # Becuse the "output" of this Unit comes from its hidden layer self.output_length = np.prod(self.output_block.shape) # additional flags if 'autoencoder' in parameters.keys() and parameters['autoencoder']: self.autoencoder = True else: self.autoencoder = False if 'complex' in parameters.keys() and parameters['complex']: self.complex = True else: self.complex = False @staticmethod def generate_missing_parameters(parameters, complex_unit=False, complex_unit_extra_layer=False, polynomial=False, autoencoder=False): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate. When complex_unit is False, a standard 3-layer MLP is used. When complex_unit is True, an MLP with additional hidden layers is used. There needs to be no return value, the method leaves a side effect by modifying the perameters dict. :param parameters: parameter dictionary :type parameters: dict """ nhidden = np.prod(parameters['output_block'].shape) ntohidden = np.prod(parameters['output_block'].shape) ninputs = 0 noutputs = 0 ncontext = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in parameters['signal_blocks']: ninputs += np.prod(block.shape)*len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS) for (block, delta, factor) in parameters['context_blocks']: # ninputs += np.prod(block.shape) ncontext += np.prod(block.shape) for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in parameters['signal_blocks']: noutputs += np.prod(block.shape) nadditional = 0 for (block, delta, pblock, pdblock) in parameters['predicted_blocks']: nadditional += np.prod(block.shape) nmiddle=nhidden*2 parameters["MLP_parameters"] = {} parameters["MLP_parameters_additional"] = {} if complex_unit: mlp_layers = [{'activation': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'error': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((ninputs+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nmiddle+ncontext+1), np.float), 'error': SharedArray.SharedNumpyArray((nmiddle+ncontext+1), np.float), 'delta': SharedArray.SharedNumpyArray((nmiddle+ncontext+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }] if complex_unit_extra_layer: mlp_layers.append({'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }) mlp_layers.append({'activation': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'error': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((noutputs+1), np.float) }) parameters["MLP_parameters"]['layers'] = mlp_layers parameters["MLP_parameters"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['beta'][0] = 1.0 parameters["MLP_parameters"]['learning_rate'] = parameters['learning_rate'] parameters["MLP_parameters"]['momentum'] = parameters['momentum'] parameters["MLP_parameters"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) mlp_weights = [MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((ninputs+1, nmiddle), np.float)), MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((nmiddle+ncontext+1, nhidden), np.float))] if complex_unit_extra_layer: mlp_weights.append(MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, nhidden), np.float))) mlp_weights.append(MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, noutputs), np.float))) parameters["MLP_parameters"]['weights'] = mlp_weights parameters["complex"] = True else: # Simple MLP Unit (not complex unit) parameters["MLP_parameters"]['layers'] = [ {'activation': SharedArray.SharedNumpyArray((ninputs+ncontext+1), np.float), 'error': SharedArray.SharedNumpyArray((ninputs+ncontext+1), np.float), 'delta': SharedArray.SharedNumpyArray((ninputs+ncontext+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'error': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((noutputs+1), np.float) }, ] parameters["MLP_parameters"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['beta'][0] = 1.0 parameters["MLP_parameters"]['learning_rate'] = parameters['learning_rate'] parameters["MLP_parameters"]['momentum'] = parameters['momentum'] parameters["MLP_parameters"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['weights'] = [ MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((ninputs+ncontext+1, nhidden), np.float)), MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, noutputs), np.float)), ] parameters["MLP_parameters_additional"]['layers'] = [ {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nadditional+1), np.float), 'error': SharedArray.SharedNumpyArray((nadditional+1), np.float), 'delta': SharedArray.SharedNumpyArray((nadditional+1), np.float) }, ] parameters["MLP_parameters_additional"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters_additional"]['beta'][0] = 1.0 parameters["MLP_parameters_additional"]['learning_rate'] = parameters['additional_learning_rate'] parameters["MLP_parameters_additional"]['momentum'] = parameters['momentum'] parameters["MLP_parameters_additional"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters_additional"]['weights'] = [ MLP.MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, nadditional), np.float)), ] if polynomial: parameters["MLP_parameters"]['polynomial'] = True parameters["MLP_parameters_additional"]['polynomial'] = True if autoencoder: parameters['autoencoder'] = True def execute0(self): """ Make predictions based on available information and publish them. """ # Prepare the input vector if not self.complex: self.layers[0]['activation'][:-1] = np.concatenate((self.inputs_t, self.contexts_t_1)) else: self.layers[0]['activation'][:-1] = self.inputs_t self.layers[1]['activation'][self.output_length*2:-1] = self.contexts_t_1 # Make prediction self.MLP.mlp_forward() self.layers_1[0]['activation'][:-1] = self.layers[self.output_layer]['activation'][:-1] self.MLP_1.mlp_forward() # Update the output self.output_block[:] = (self.layers[self.output_layer]['activation'][:self.output_length]).reshape(self.output_block.shape) # Publish predictions (mainly for debugging and inspection) i = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in self.signal_blocks: l = np.prod(pred_block.shape) pred_block[:] = (self.inputs_t[i:i+l]).reshape(pred_block.shape) pred_block_local[:] = (self.inputs_t[i:i+l]).reshape(pred_block_local.shape) i += 4*l i = 0 for (block, delta, predicted_block, predicted_d_block) in self.predicted_blocks: l = np.prod(predicted_block.shape) predicted_block[:] = (self.inputs_t[i:i+l]).reshape(predicted_block.shape) i += l def execute1(self): """ Collect new signals, perform association, prepare for additional learning """ # Collect the current signal i = 0 j = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in self.signal_blocks: l = np.prod(block.shape) # Calculate signal features e_block = 0.5*(block-pred_block_local)+0.5 # Rescale so it could possibly be predicted by sigmoid units dblock[:] = 0.5*(block-past_block)+0.5 # Rescale so it could possibly be predicted by sigmoid units past_block[:] = block iblock[:] = self.tau[0] * iblock + (1-self.tau[0]) * block # Proportional signal self.inputs_t[i:i + l] = block.flatten() # Derivative signal self.inputs_t[i + l:i + 2 * l] = dblock.flatten() # Integral signal self.inputs_t[i + 2 * l:i + 3 * l] = iblock.flatten() # Prediction Error signal self.inputs_t[i + 3 * l:i + 4 * l] = e_block.flatten() i += len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS)*l # For autoencoder we keep the previous signal if not self.autoencoder: self.predictions_t[j:j+l] = block.flatten() j += l i = 0 for (block, delta, pblock, pdblock) in self.predicted_blocks: l = np.prod(block.shape) self.pinputs_t[i:i + l] = block.flatten() i += l if self.learning_rate[0] != 0: # Calculate the prediction error error_x = self.predictions_t - self.layers[-1]['activation'][:np.prod(self.predictions_t.shape)] # Make the association self.MLP.train2(error=error_x) # Extract the new deltas for input blocks i = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in self.signal_blocks: l = np.prod(block.shape) delta[:] = self.layers[0]['delta'][i:i + l].reshape(delta.shape) i += len(ExecutionUnit.UNSUPERVISED_SIGNAL_INPUTS)*l if self.learning_rate_1[0] != 0: error_p = self.pinputs_t - self.layers_1[-1]['activation'][:-1] self.MLP_1.train2(error=error_p) # For autoencoder new signal is collected after the training is done if self.autoencoder: j = 0 for (block, delta, pred_block, past_block, dblock, iblock, pred_block_local) in self.signal_blocks: l = np.prod(block.shape) self.predictions_t[j:j+l] = block.flatten() j += l def execute2(self): """ Gather the context signal in preparation for the next step. Some context blocks get their information from units lateral to this one, others get their information from units above this one. The "factor" is either the context_factor_lateral or context_factor_feedback """ # Prepare information for the next step # collect the new context i = 0 for (block, delta, factor) in self.context_blocks: l = np.prod(block.shape) self.contexts_t_1[i:i+l] = factor[0]*block.flatten() i += l def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/PVM_unit_v1.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP import copy import logging class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): """ Unit architecture: [predicted block],[predicted block], ... ,[predicted pblock],[predicted pblock] \ / \ / \ / [ compressed features - output block ] / \ / \ / \ [block][eblock][dblock][iblock],[block] ..., [context block][context block] Features used by the unit to predict each block: - previous value of the block - error in previous prediction - temporal derivative of the block - integral of the block - ? """ # signal_block, delta_block, prediction_t+1, prediction_t+2 SIGNAL_BLOCK_CONTENTS = set(["signal_block: Raw signal block", "delta_block: Delta extracted from associator", "prediction_tp1: Predicted block at t+1", "prediction_tp2: Predicted block at t+2"]) UNSUPERVISED_SIGNAL_INPUTS = set(["block: Raw signal block t-1" ]) UNSUPERVISED_CONTEXT_INPUTS = set(["block: Raw feedback or context input Block", "delta: Delta of the context inputs [currently NOT computed or used]", "factor: Factor that scales this feedback or context input"]) # Each block: teaching_signal, delta_block, readout_block SUPERVISED_TASK_OUTPUTS = set(["teaching_block: Target heatmap", "delta: Prediction error of heatmap", "readout_block: Prediction of heatmap"]) @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 3 # because there is execute0, 1 and 2 implemented def open_views_into_shmem(self, shared_memory, key, num_items): return map(lambda x: tuple([x[i].view(np.ndarray) for i in range(num_items)]), shared_memory[key]) def open_views_into_shmem_single(self, shared_memory, key): return map(lambda x: x.view(np.ndarray), shared_memory[key]) def __init__(self, parameters): # Mapping SharedArray items through .view(np.ndarray) improved the access time self.signal_blocks = self.open_views_into_shmem(parameters, "signal_blocks", len(ExecutionUnit.SIGNAL_BLOCK_CONTENTS)) self.readout_blocks = self.open_views_into_shmem(parameters, "readout_blocks", len(ExecutionUnit.SUPERVISED_TASK_OUTPUTS)) self.context_blocks = self.open_views_into_shmem(parameters, "context_blocks", len(ExecutionUnit.UNSUPERVISED_CONTEXT_INPUTS)) self.derivative_blocks = self.open_views_into_shmem_single(parameters, "derivative_blocks") self.delta_blocks = self.open_views_into_shmem_single(parameters, "delta_blocks") self.integral_blocks = self.open_views_into_shmem_single(parameters, "integral_blocks") self.error_blocks = self.open_views_into_shmem_single(parameters, "error_blocks") self.internal_buffers = self.open_views_into_shmem(parameters, "internal_buffers", 3) self.output_block = parameters['output_block'].view(np.ndarray) # Will appear in other places as either input or context self.output_min = parameters['output_min'] self.output_max = parameters['output_max'] self.avg_delta = parameters['avg_delta'] self.internal_buffers = parameters['internal_buffers'] self.flags = parameters['flags'] # The perceptron self.MLP_internal_prediction = MLP.MLP(parameters["Primary_Predictor_params"]) self.MLP_readout = MLP.MLP(parameters["Readout_Predictor_params"]) # This is a "task" supervised MLP (e.g., tracker) self.primary_learning_rate = parameters['primary_learning_rate'] self.readout_learning_rate = parameters['readout_learning_rate'] self.layers_internal_prediction = parameters["Primary_Predictor_params"]['layers'] self.layers_readout = parameters["Readout_Predictor_params"]['layers'] # These are the "task" supervised MLP layers self.tau = parameters['tau'] # Tau is the integration constant for the signal integral # Operation parameters self.input_blocks_skip = 1 self.output_blocks_skip = 1 self.backpropagate_readout_error = False self.complex = False self.complex_context_in_second_layer = False self.use_global_backprop = False self.use_t_2_block = False self.use_derivative = False self.use_error = False self.use_integral = False self.predict_2_steps = False self.normalize = False if 'backpropagate_readout_error' in parameters.keys(): self.normalize = parameters['backpropagate_readout_error'] if 'normalize_output' in parameters.keys(): self.normalize = parameters['normalize_output'] if 'complex' in parameters.keys(): self.complex = parameters['complex'] if 'complex_context_in_second_layer' in parameters.keys(): self.complex_context_in_second_layer = parameters['complex_context_in_second_layer'] if 'use_derivative' in parameters.keys(): self.use_derivative = parameters['use_derivative'] if 'use_integral' in parameters.keys(): self.use_integral = parameters['use_integral'] if 'use_error' in parameters.keys(): self.use_error = parameters['use_error'] if 'use_t_2_block' in parameters.keys(): self.use_t_2_block = parameters['use_t_2_block'] if 'use_global_backprop' in parameters.keys(): self.use_global_backprop = parameters['use_global_backprop'] if 'autoencoder' in parameters.keys(): self.autoencoder = parameters['autoencoder'] if 'predict_2_steps' in parameters.keys(): self.predict_2_steps = parameters['predict_2_steps'] if self.use_derivative: self.input_blocks_skip += 1 if self.use_integral: self.input_blocks_skip += 1 if self.use_error: self.input_blocks_skip += 1 if self.use_t_2_block: self.input_blocks_skip += 1 if self.predict_2_steps: self.output_blocks_skip += 1 # Input buffers self.ninputs = 0 self.npredictions = 0 self.npinputs = 0 self.ncontexts = 0 for (block, delta, pred_block1, pred_block2) in self.signal_blocks: self.ninputs += self.input_blocks_skip * np.prod(block.shape) self.npredictions += self.output_blocks_skip * np.prod(block.shape) for (teaching_block, delta, readout_block) in self.readout_blocks: self.npinputs += np.prod(teaching_block.shape) self.inputs_t = np.zeros((self.ninputs,)) self.actual_signal_t = np.zeros((self.npredictions/self.output_blocks_skip, )) self.readout_training_signal = np.zeros((self.npinputs,)) self.inputs_t_1 = np.zeros((self.ninputs,)) self.actual_signal_t_1 = np.zeros((self.npredictions/self.output_blocks_skip, )) self.pinputs_t_1 = np.zeros((self.npinputs,)) # Context buffers for (block, delta, factor) in self.context_blocks: self.ncontexts += np.prod(block.shape) self.contexts_t_1 = np.zeros((self.ncontexts,)) self.output_layer = len(self.layers_internal_prediction)-2 # Becuse the "output" of this Unit comes from its hidden layer self.output_length = np.prod(self.output_block.shape) # Buffer for storing activations self.activations_buffer = [] # additional flags if self.use_global_backprop: self.push_activation() if self.predict_2_steps: self.push_activation() # If two steps into the future are being predicted the training can only be done once all the data is avaliable # and therefore older activations need to be recovered. # Similarly in the case of the global backprop, the deltas computed on the current step correspnd to activations # from one step behind therefore again the state of the network needts to be rewinded one step back for training # In the case of using both global backprop and predicting two steps into the future things are a it more complicated. # First the training needts to be applied to the state of the network from one step behind. At this stage the deltas # are computed bu now these deltas belong to the state two steps behind, so for global backprop step the # state of the network needs to be rewinded one step further. # Since that is so, one needs to keep a queue of 3 states. @staticmethod def upgrade_to_ver_1(parameters): parameters['internal_buffers'] = [] parameters['output_min'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['output_max'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['avg_delta'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['integral_blocks'] = [] parameters['derivative_blocks'] = [] parameters['error_blocks'] = [] parameters['use_derivative'] = True parameters['use_integral'] = True parameters['use_error'] = True parameters['use_t_2_block'] = False parameters['predict_2_steps'] = False parameters['use_global_backprop'] = False parameters['normalize_output'] = False parameters["complex_context_in_second_layer"] = False for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: block01 = SharedArray.SharedNumpyArray_like(block) block02 = SharedArray.SharedNumpyArray_like(block) block03 = SharedArray.SharedNumpyArray_like(block) parameters['internal_buffers'].append((block01, block02, block03)) parameters['derivative_blocks'].append(SharedArray.SharedNumpyArray_like(block)) parameters['integral_blocks'].append(SharedArray.SharedNumpyArray_like(block)) parameters['error_blocks'].append(SharedArray.SharedNumpyArray_like(block)) if "complex" not in parameters.keys(): parameters["complex"] = False if len(parameters["Primary_Predictor_params"]['layers']) == 4: parameters["complex"] = True if "autoencoder" not in parameters.keys(): parameters["autoencoder"] = False if "readout_learning_rate" not in parameters.keys(): parameters['readout_learning_rate'] = parameters["Primary_Predictor_params"]["learning_rate"] if "momentum" not in parameters.keys(): parameters['momentum'] = parameters["Primary_Predictor_params"]["momentum"] nhidden = parameters["Primary_Predictor_params"]['layers'][-2]['activation'].shape[0]-1 nreadout = 0 nouputs = 0 for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: nouputs += np.prod(block.shape) for (block, delta, pblock) in parameters['readout_blocks']: nreadout += np.prod(block.shape) if "Readout_Predictor_params" not in parameters.keys(): parameters["Readout_Predictor_params"] = {} parameters["Readout_Predictor_params"]['layers'] = MLP.get_layers([nhidden+1, nreadout+1]) parameters["Readout_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['beta'][0] = 1.0 parameters["Readout_Predictor_params"]['learning_rate'] = parameters['readout_learning_rate'] parameters["Readout_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Readout_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['weights'] = MLP.get_weights(parameters["Readout_Predictor_params"]['layers']) parameters["Readout_Predictor_params"]['weights'][0][:] = parameters["Primary_Predictor_params"]['weights'][-1][:, nouputs:] old_weight_matrix = parameters["Primary_Predictor_params"]['weights'][-1] parameters["Primary_Predictor_params"]['weights'][-1] = SharedArray.SharedNumpyArray((nhidden+1, nouputs), np.float) parameters["Primary_Predictor_params"]['weights'][-1][:] = old_weight_matrix[:, :nouputs] parameters["Primary_Predictor_params"]['layers'][-1] = {'activation': SharedArray.SharedNumpyArray(nouputs, np.float), 'error': SharedArray.SharedNumpyArray(nouputs, np.float), 'delta': SharedArray.SharedNumpyArray(nouputs, np.float) } parameters['backpropagate_readout_error'] = True @staticmethod def generate_missing_parameters(parameters, options): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate. When complex_unit is False, a standard 3-layer MLP is used. When complex_unit is True, an MLP with additional hidden layers is used. There needs to be no return value, the method leaves a side effect by modifying the perameters dict. :param parameters: parameter dictionary :type parameters: dict """ complex_unit = options['unit_type'] == "complex" polynomial = options['polynomial'] == '1' autoencoder = options['autoencoder'] == '1' use_t_2_block = options['use_t_minus_2_block'] == '1' use_derivative = options['use_derivative'] == '1' use_integral = options['use_integral'] == '1' use_error = options['use_error'] == '1' predict_2_steps = options['predict_two_steps'] == '1' use_global_backprop = options['use_global_backprop'] == '1' complex_context_in_second_layer = options['feed_context_in_complex_layer'] == '1' parameters['normalize_output'] = options["normalize_output"] == "1" parameters['backpropagate_readout_error'] = options["backpropagate_readout_error"] == "1" nhidden = np.prod(parameters['output_block'].shape) parameters['output_min'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) parameters['output_max'] = SharedArray.SharedNumpyArray_like(parameters['output_block'])+1 parameters['avg_delta'] = SharedArray.SharedNumpyArray_like(parameters['output_block']) ninputs = 0 noutputs = 0 ncontext = 0 # Any additional memory buffers needed in the operation of the unit parameters['internal_buffers'] = [] parameters['integral_blocks'] = [] parameters['derivative_blocks'] = [] parameters['error_blocks'] = [] parameters['use_derivative'] = use_derivative parameters['use_integral'] = use_integral parameters['use_error'] = use_error parameters['use_t_2_block'] = use_t_2_block parameters['predict_2_steps'] = predict_2_steps for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: block01 = SharedArray.SharedNumpyArray_like(block) block02 = SharedArray.SharedNumpyArray_like(block) block03 = SharedArray.SharedNumpyArray_like(block) parameters['internal_buffers'].append((block01, block02, block03)) if use_derivative: parameters['derivative_blocks'].append(SharedArray.SharedNumpyArray_like(block)) if use_integral: parameters['integral_blocks'].append(SharedArray.SharedNumpyArray_like(block)) if use_error: parameters['error_blocks'].append(SharedArray.SharedNumpyArray_like(block)) input_block_features = 1 output_predictions = 1 if use_derivative: input_block_features += 1 if use_integral: input_block_features += 1 if use_error: input_block_features += 1 if use_t_2_block: input_block_features += 1 if predict_2_steps: output_predictions += 1 for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: ninputs += np.prod(block.shape) * input_block_features for (block, delta, factor) in parameters['context_blocks']: ncontext += np.prod(block.shape) for (block, delta, pred_block, pred_block2) in parameters['signal_blocks']: noutputs += np.prod(block.shape) * output_predictions nreadout = 0 for (block, delta, pblock) in parameters['readout_blocks']: nreadout += np.prod(block.shape) parameters["Primary_Predictor_params"] = {} parameters["Readout_Predictor_params"] = {} if complex_unit and complex_context_in_second_layer: # 4 layer perceptron parameters["Primary_Predictor_params"]['layers'] = MLP.get_layers([ninputs+1, 2*nhidden+ncontext+1, nhidden+1, noutputs+1]) elif complex_unit: parameters["Primary_Predictor_params"]['layers'] = MLP.get_layers([ninputs+ncontext+1, 2*nhidden+1, nhidden+1, noutputs+1]) else: # 3 layer perceptron Simple MLP Unit (not complex unit) parameters["Primary_Predictor_params"]['layers'] = MLP.get_layers([ninputs+ncontext+1, nhidden+1, noutputs+1]) parameters["Primary_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Primary_Predictor_params"]['beta'][0] = 1.0 parameters["Primary_Predictor_params"]['learning_rate'] = parameters['primary_learning_rate'] parameters["Primary_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Primary_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Primary_Predictor_params"]['weights'] = MLP.get_weights(parameters["Primary_Predictor_params"]['layers']) parameters["Readout_Predictor_params"]['layers'] = MLP.get_layers([nhidden+1, 2*nhidden+1, nreadout+1]) parameters["Readout_Predictor_params"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['beta'][0] = 1.0 parameters["Readout_Predictor_params"]['learning_rate'] = parameters['readout_learning_rate'] parameters["Readout_Predictor_params"]['momentum'] = parameters['momentum'] parameters["Readout_Predictor_params"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["Readout_Predictor_params"]['weights'] = MLP.get_weights(parameters["Readout_Predictor_params"]['layers']) parameters["Primary_Predictor_params"]['polynomial'] = polynomial parameters["Readout_Predictor_params"]['polynomial'] = polynomial parameters['autoencoder'] = autoencoder parameters['use_global_backprop'] = use_global_backprop parameters["complex_context_in_second_layer"] = complex_context_in_second_layer parameters["complex"] = complex_unit def min_max_normalize(self, a, min_a, max_a): """ Simple min-max normalization :param a: array to be normalized :param min_a: recorded min values :param max_a: recorded max values :return: """ if self.primary_learning_rate[0] != 0: min_a[:] = np.minimum(a, min_a) + 0.000001 max_a[:] = np.maximum(a, max_a) - 0.000001 return np.divide(a-min_a, max_a-min_a) def execute0(self): """ Make predictions based on available information and publish them. """ # Prepare the input vector if self.complex and self.complex_context_in_second_layer: self.layers_internal_prediction[0]['activation'][:-1] = self.inputs_t self.layers_internal_prediction[1]['activation'][-(self.ncontexts+1):-1] = self.contexts_t_1 else: self.layers_internal_prediction[0]['activation'][:-1] = np.concatenate((self.inputs_t, self.contexts_t_1)) # Make internal prediction self.MLP_internal_prediction.mlp_forward() # Forward internal representations to the readout predictor self.layers_readout[0]['activation'][:-1] = self.layers_internal_prediction[self.output_layer]['activation'][:-1] # Make readout prediction self.MLP_readout.mlp_forward() # Update the output if self.normalize: self.output_block[:] = self.min_max_normalize((self.layers_internal_prediction[self.output_layer]['activation'][:self.output_length]).reshape(self.output_block.shape), self.output_min, self.output_max) else: self.output_block[:] = (self.layers_internal_prediction[self.output_layer]['activation'][:self.output_length]).reshape(self.output_block.shape) # Publish predictions (mainly for debugging and inspection) i = 0 s = len(self.signal_blocks) for (idx, (block, delta, pred_block1, pred_block2)) in enumerate(self.signal_blocks): l = np.prod(pred_block1.shape) pred_block1[:] = np.clip((self.layers_internal_prediction[-1]['activation'][i:i+l]).reshape(pred_block1.shape), 0, 1) pred_block_local = self.internal_buffers[idx][0] pred_block_local[:] = np.clip((self.layers_internal_prediction[-1]['activation'][i:i+l]).reshape(pred_block_local.shape), 0, 1) if self.predict_2_steps: pred_block2[:] = np.clip((self.layers_internal_prediction[-1]['activation'][i+l*s:i+l*s+l]).reshape(pred_block1.shape), 0, 1) i += l i = 0 for (block, delta, readout_block) in self.readout_blocks: l = np.prod(readout_block.shape) readout_block[:] = (self.layers_readout[-1]['activation'][i:i+l]).reshape(readout_block.shape) i += l # Since predicting two steps into the future, learning can only be performed on previous # activations, hence current activations are pushed into the FIFO if (self.use_global_backprop or self.predict_2_steps) and self.primary_learning_rate[0] != 0: self.push_activation() def execute1(self): """ Collect new signals, perform association, prepare for additional learning """ # Collect the current signal curent_i = 0 j = 0 for (idx, (input_block, delta, pred_block1, pred_block2)) in enumerate(self.signal_blocks): if self.flags[0] != 12: block = input_block else: block = pred_block1 l = np.prod(block.shape) forward_i = 0 # Calculate signal features past_block = self.internal_buffers[idx][1] self.inputs_t[curent_i:curent_i + l] = block.flatten() curent_i += l if self.use_derivative: self.derivative_blocks[idx][:] = 0.5*(block - past_block)+0.5 self.inputs_t[curent_i:curent_i+l] = self.derivative_blocks[idx].flatten() curent_i += l if self.use_integral: self.integral_blocks[idx][:] = self.tau[0] * self.integral_blocks[idx] + (1 - self.tau[0]) * block self.inputs_t[curent_i:curent_i+l] = self.integral_blocks[idx].flatten() curent_i += l if self.use_error: pred_block_local = self.internal_buffers[idx][0] self.error_blocks[idx][:] = 0.5*(block - pred_block_local)+0.5 self.inputs_t[curent_i:curent_i+l] = self.error_blocks[idx].flatten() curent_i += l if self.use_t_2_block: self.inputs_t[curent_i:curent_i+l] = past_block.flatten() curent_i += l past_block[:] = block # Collect the current signal as supervising signal, unless running as an autoencoder if not self.autoencoder: self.actual_signal_t[j:j+l] = block.flatten() j += l # Predictive associative training if self.primary_learning_rate[0] != 0: # pop the previous activaitons from the FIFO if self.predict_2_steps: if self.use_global_backprop: self.get_one_step_behind_activation() else: self.pop_activation() training_signal = np.concatenate((self.actual_signal_t_1, self.actual_signal_t)) else: training_signal = self.actual_signal_t # Calculate the prediction error error_primary = training_signal - self.layers_internal_prediction[-1]['activation'][:np.prod(training_signal.shape)] self.actual_signal_t_1[:] = self.actual_signal_t # Make the association self.MLP_internal_prediction.train2(error=error_primary) if self.use_global_backprop: i = 0 # Extract the deltas for input blocks: for (block, delta, pred_block1, pred_block2) in self.signal_blocks: l = np.prod(block.shape) delta[:] = self.layers_internal_prediction[0]['delta'][i:i+l].reshape(delta.shape) i += l * self.input_blocks_skip if self.complex and self.complex_context_in_second_layer: base_idx = self.layers_internal_prediction[1]['delta'].shape[0]-self.ncontexts-1 for (idx, (block, delta, factor)) in enumerate(self.context_blocks): if factor[0] > 0: l = np.prod(block.shape) delta[:] = self.layers_internal_prediction[1]['delta'][base_idx + idx * l:base_idx + (idx+1) * l].reshape(delta.shape) else: for (block, delta, factor) in self.context_blocks: if factor[0] > 0: l = np.prod(block.shape) delta[:] = self.layers_internal_prediction[0]['delta'][i:i + l].reshape(delta.shape) i += l # Readout training if self.readout_learning_rate[0] != 0: # Collect the curent readout training signal i = 0 for (block, delta, pblock) in self.readout_blocks: l = np.prod(block.shape) self.readout_training_signal[i:i + l] = block.flatten() i += l error_readout = self.readout_training_signal - self.layers_readout[-1]['activation'][:-1] self.MLP_readout.train2(error=error_readout) if self.backpropagate_readout_error and self.primary_learning_rate[0] != 0: # If true the error generated by the readout will be propagated into the predictive encoder self.layers_internal_prediction[self.output_layer]['delta'][:-1] = self.layers_readout[0]['delta'][:-1] for l in range(self.output_layer-1, -1, -1): self.MLP_internal_prediction.mlp_backpropagate_layer(layer=l) for l in range(self.output_layer-1, -1, -1): self.MLP_internal_prediction.calculate_weight_update_layer(layer=l) self.MLP_internal_prediction.update_weights_layer(layer=l) if self.autoencoder: j = 0 for (idx, (block, delta, pred_block1, pred_block2)) in enumerate(self.signal_blocks): l = np.prod(block.shape) self.actual_signal_t[j:j+l] = block.flatten() j += l def execute2(self): """ Gather the context signal in preparation for the next step. Some context blocks get their information from units lateral to this one, others get their information from units above this one. The "factor" is either the context_factor_lateral or context_factor_feedback """ if self.primary_learning_rate[0] != 0 and self.use_global_backprop: self.pop_activation() self.avg_delta *= 0 for delta in self.delta_blocks: self.avg_delta += delta self.avg_delta /= len(self.delta_blocks) self.layers_internal_prediction[self.output_layer]['delta'][:-1] = self.avg_delta.flatten() for l in range(self.output_layer-1, -1, -1): self.MLP_internal_prediction.mlp_backpropagate_layer(layer=l) for l in range(self.output_layer-1, -1, -1): self.MLP_internal_prediction.calculate_weight_update_layer(layer=l) self.MLP_internal_prediction.update_weights_layer(layer=l) # Prepare information for the next step # collect the new context i = 0 for (block, delta, factor) in self.context_blocks: l = np.prod(block.shape) self.contexts_t_1[i:i+l] = factor[0]*block.flatten() i += l def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass def push_activation(self): """ Save the current network activations to a FIFO queue :return: """ layers_copy = copy.deepcopy(self.layers_internal_prediction) self.activations_buffer.append(layers_copy) def pop_activation(self): """ Pop the first available activations from the FIFO :return: """ layers = self.activations_buffer.pop(0) for layer in range(len(layers)): self.layers_internal_prediction[layer]['activation'][:] = layers[layer]['activation'] self.layers_internal_prediction[layer]['error'][:] = layers[layer]['error'] self.layers_internal_prediction[layer]['delta'][:] = layers[layer]['delta'] def get_one_step_behind_activation(self): """ Pop the first available activations from the FIFO :return: """ layers = self.activations_buffer[-2] for layer in range(len(layers)): self.layers_internal_prediction[layer]['activation'][:] = layers[layer]['activation'] self.layers_internal_prediction[layer]['error'][:] = layers[layer]['error'] self.layers_internal_prediction[layer]['delta'][:] = layers[layer]['delta'] ================================================ FILE: PVM_models/__init__.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== ================================================ FILE: PVM_models/demo00_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time import cv2 import numpy as np import PVM_framework.PVM_Create as PVM_Create import PVM_framework.CoreUtils as CoreUtils import PVM_framework.SharedArray as SharedArray import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import os import logging import importlib class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True def start(self): """ This will be called right before the simulation starts """ self.t_start = time.time() self._running = True self.steps = 0 def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ pass def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ cv2.imshow("arena", self.prop_dict['arena'].view(np.ndarray)) cv2.waitKey(1) if self.steps > self.steps_to_run: self._running = False now = time.time() self.steps += 1 sps = self.steps / (now-self.t_start) print ("%3.3f steps per sec" % (sps)) + "\r", def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass def generate_dict(): # simulation dictionary has to have: # N - SharedArray of one entity enumerating the step of the simulation # stages - integer number of processing stages # num_proc - integer number of processors that will be utilized for workers # stage0 - a list of element parameters (dictionaries) for each execution element # stage0_size - number of execution elements in the stage # other parameters are model specific. simulation_dict = PVM_Create.create_blank_dictionary() simulation_dict['stages'] = 1 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = 'PVM_models.demo00_unit' unit = importlib.import_module(simulation_dict['execution_unit_module']) simulation_dict['stage0_size'] = 100 arena = SharedArray.SharedNumpyArray((500, 500), np.uint8) # the arena will be 500x500 in this case simulation_dict['arena'] = arena for i in range(simulation_dict['stage0_size']): unit_parameters = {} unit_parameters['arena'] = arena # block will define a sub-block of the arena that each unit will be processing # it will contain (x0, y0, width, height) unit_parameters['block'] = SharedArray.SharedNumpyArray((4,), np.int) # each unit gets a 100x100 sub-block of the arena unit_parameters['block'][0] = (i / 10)*50 unit_parameters['block'][1] = (i % 10)*50 unit_parameters['block'][2] = 50 unit_parameters['block'][3] = 50 simulation_dict['stage0'].append(unit_parameters) for i in range(simulation_dict['stage0_size']): unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i]) return simulation_dict def run_demo(): """ In this very simple demo a set of workers operate on a 500x500 image domain by randomly flipping selected bits. To make things faster the bit/byte flipping function is written in cython. :return: """ filename = "demo00_state.p.gz" if os.path.isfile(filename): state_dict = CoreUtils.load_model(filename) else: state_dict = generate_dict() manager = Manager(state_dict, 1000) executor = CoreUtils.ModelExecution(prop_dict=state_dict, manager=manager) executor.start(blocking=True) CoreUtils.save_model(state_dict, filename) print("Saving and running again in non blocking mode") executor.start(blocking=False) while manager.running(): executor.step() executor.finish() CoreUtils.save_model(state_dict, filename) if __name__ == '__main__': logging.basicConfig(filename="demo00.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s') run_demo() ================================================ FILE: PVM_models/demo00_unit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.fast_routines as fast_routines class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods """ return 1 # because there is only execute0 implemented def __init__(self, parameters): self.arena = parameters['arena'] self.block = parameters['block'] @staticmethod def generate_missing_parameters(parameters): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate :return: """ return parameters def execute0(self): """ This method does the actual execution. """ # Using cythonized code, runs much faster fast_routines.flip_some_bytes(100, self.arena.view(np.ndarray), self.block.view(np.ndarray)) def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/demo01_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time import cv2 import numpy as np import PVM_framework.CoreUtils as CoreUtils import PVM_framework.PVM_Create as PVM_Create import PVM_framework.SharedArray as SharedArray import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import os import logging import importlib class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True def start(self): """ This will be called right before the simulation starts """ self.t_start = time.time() self.steps = 0 def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ pass def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ image = (self.prop_dict['arena']+1).astype(np.uint8)*127 cv2.imshow("arena", image) key = cv2.waitKey(1) & 0xFF if key == 27 or self.steps > self.steps_to_run: self._running = False now = time.time() self.steps += 1 sps = self.steps / (now-self.t_start) print ("%3.3f steps per sec" % (sps)) + "\r", def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass def generate_dict(): # simulation dictionary has to have: # N - SharedArray of one entity enumerating the step of the simulation # stages - integer number of processing stages # num_proc - integer number of processors that will be utilized for workers # stage0 - a list of element parameters (dictionaries) for each execution element # stage0_size - number of execution elements in the stage # other parameters are model specific. simulation_dict = PVM_Create.create_blank_dictionary() simulation_dict['stages'] = 1 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = 'PVM_models.demo01_unit' unit = importlib.import_module(simulation_dict['execution_unit_module']) simulation_dict['stage0_size'] = 100 arena = SharedArray.SharedNumpyArray((1000, 1000), np.int8) # the arena will be 1000x1000 pixels in this case arena.copyto(2*np.random.randint(2, size=arena.shape).astype(np.int8)-1) simulation_dict['arena'] = arena temperature = SharedArray.SharedNumpyArray((1,), np.float) temperature[0] = 1/2.269 simulation_dict['temperature'] = temperature for i in range(simulation_dict['stage0_size']): unit_parameters = {} unit_parameters['arena'] = arena unit_parameters['temperature'] = temperature # block will define a sub-block of the arena that each unit will be processing # it will contain (x0, y0, width, height) unit_parameters['block'] = SharedArray.SharedNumpyArray((4,), np.int) # each unit gets a 100x100 sub-block of the arena unit_parameters['block'][0] = (i / 10)*100 unit_parameters['block'][1] = (i % 10)*100 unit_parameters['block'][2] = 100 unit_parameters['block'][3] = 100 simulation_dict['stage0'].append(unit_parameters) for i in range(simulation_dict['stage0_size']): unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i]) return simulation_dict def run_demo(): """ In this simple demo the crticial temperature Ising model is run synchronously by a set of units on a large 1000x1000 domain. To make things fast the worker code if written in cython. :return: """ filename = "demo01_state.p.gz" if os.path.isfile(filename): sate_dict = CoreUtils.load_model(filename) else: sate_dict = generate_dict() manager = Manager(sate_dict, 1000000) CoreUtils.run_model(sate_dict, manager) CoreUtils.save_model(sate_dict, filename) if __name__ == '__main__': logging.basicConfig(filename="demo01.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s') logging.getLogger().addHandler(logging.StreamHandler()) # Send logs to stdout run_demo() ================================================ FILE: PVM_models/demo01_unit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.fast_routines as fast_routines class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 1 # because there is only execute0 implemented def __init__(self, parameters): self.arena = parameters['arena'] self.block = parameters['block'] self.temperature = parameters['temperature'] @staticmethod def generate_missing_parameters(parameters): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate """ return parameters def execute0(self): """ This method does the actual execution. """ # Using cythonized code, runs much faster fast_routines.ising_model(100, self.arena.view(np.ndarray), self.block.view(np.ndarray), self.temperature[0]) def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/demo02_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time import cv2 import numpy as np import PVM_framework.PVM_Create as PVM_Create import PVM_framework.CoreUtils as CoreUtils import PVM_framework.SharedArray as SharedArray import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import os import logging import importlib import argparse class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run, cam): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True self.cam = cam self.steps = 0 def start(self): """ This will be called right before the simulation starts """ self.t_start = time.time() ret, frame = self.cam.read() self.yet_previous_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) ret, frame = self.cam.read() self.previous_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) ret, frame = self.cam.read() self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ # switch buffers self.prop_dict['yet_previous_array'].copyto(self.yet_previous_frame) self.prop_dict['previous_array'].copyto(self.previous_frame) self.prop_dict['current_array'].copyto(self.current_frame) pass def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ ret, frame = self.cam.read() self.yet_previous_frame = self.previous_frame self.previous_frame = self.current_frame self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) cv2.imshow("Current frame", self.prop_dict['current_array'].view(np.ndarray)) cv2.imshow("Previous frame", self.prop_dict['previous_array'].view(np.ndarray)) cv2.imshow("Predicted frame", self.prop_dict['predicted_array'].view(np.ndarray)) cv2.imshow("Difference", cv2.absdiff(self.prop_dict['predicted_array'].view(np.ndarray), self.prop_dict['current_array'].view(np.ndarray))) key = cv2.waitKey(1) & 0xFF if key == 27 or self.steps > self.steps_to_run: self._running = False now = time.time() self.steps += 1 sps = self.steps / (now-self.t_start) print ("%3.3f steps per sec" % (sps)) + "\r", def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass def generate_dict(): # simulation dictionary has to have: # N - SharedArray of one entity enumerating the step of the simulation # stages - integer number of processing stages # num_proc - integer number of processors that will be utilized for workers # stage0 - a list of element parameters (dictionaries) for each execution element # stage0_size - number of execution elements in the stage # other parameters are model specific. simulation_dict = PVM_Create.create_blank_dictionary() simulation_dict['stages'] = 1 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = 'PVM_models.demo02_unit' unit = importlib.import_module(simulation_dict['execution_unit_module']) blocks_per_dim = 40 block_size = 5 simulation_dict['stage0_size'] = blocks_per_dim*blocks_per_dim yet_previous_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['yet_previous_array'] = yet_previous_array previous_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['previous_array'] = previous_array current_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['current_array'] = current_array predicted_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['predicted_array'] = predicted_array learning_rate = SharedArray.SharedNumpyArray((1,), np.float) learning_rate[0] = 0.01 simulation_dict['learning_rate'] = learning_rate momentum = SharedArray.SharedNumpyArray((1,), np.float) momentum[0] = 0.5 simulation_dict['momentum'] = momentum for i in range(simulation_dict['stage0_size']): unit_parameters = {} unit_parameters['learning_rate'] = learning_rate unit_parameters['momentum'] = momentum unit_parameters['yet_previous_array'] = yet_previous_array unit_parameters['previous_array'] = previous_array unit_parameters['current_array'] = current_array unit_parameters['predicted_array'] = predicted_array # block will define a sub-block of the arena that each unit will be processing # it will contain (x0, y0, width, height) unit_parameters['block'] = SharedArray.SharedNumpyArray((4,), np.int) # each unit gets a 100x100 sub-block of the arena unit_parameters['block'][0] = (i / blocks_per_dim)*block_size unit_parameters['block'][1] = (i % blocks_per_dim)*block_size unit_parameters['block'][2] = block_size unit_parameters['block'][3] = block_size simulation_dict['stage0'].append(unit_parameters) for i in range(simulation_dict['stage0_size']): unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i]) return simulation_dict def run_demo(movie_file): """ In this demo a simple future/predictive encoder is being instantiated to predict a camera image based on two previous frames. """ filename = "demo02_state.p.gz" if movie_file != "": cam = cv2.VideoCapture(movie_file) else: cam = cv2.VideoCapture(-1) if not cam.isOpened(): logging.error("Either cannot read the input file or no camera found!") exit(1) if os.path.isfile(filename): dict = CoreUtils.load_model(filename) else: dict = generate_dict() manager = Manager(dict, 1000, cam=cam) CoreUtils.run_model(dict, manager) CoreUtils.save_model(dict, filename) if __name__ == '__main__': logging.basicConfig(filename="demo02.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s') logging.getLogger().addHandler(logging.StreamHandler()) # Send logs to stdout parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="Movie file to load. If empty the script will try to use the camera", type=str, default="") args = parser.parse_args() run_demo(args.file) ================================================ FILE: PVM_models/demo02_unit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 1 # because there is only execute0 implemented def __init__(self, parameters): self.yet_previous_array = parameters['yet_previous_array'].view(np.ndarray) self.previous_array = parameters['previous_array'].view(np.ndarray) self.current_array = parameters['current_array'].view(np.ndarray) self.predicted_array = parameters['predicted_array'].view(np.ndarray) self.block = parameters['block'].view(np.ndarray) self.MLP = MLP.MLP(parameters["MLP_parameters"]) @staticmethod def generate_missing_parameters(parameters): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate """ ninputs = 2*parameters['block'][2]*parameters['block'][3]*3 nhidden = ninputs / 5 noutputs = parameters['block'][2]*parameters['block'][3]*3 parameters["MLP_parameters"] = {} parameters["MLP_parameters"]['layers'] = [ {'activation': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'error': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((ninputs+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'error': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((noutputs+1), np.float) }, ] parameters["MLP_parameters"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['beta'][0] = 1.0 parameters["MLP_parameters"]['learning_rate'] = parameters['learning_rate'] parameters["MLP_parameters"]['momentum'] = parameters['momentum'] parameters["MLP_parameters"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['weights'] = [ MLP.initialize_weights(SharedArray.SharedNumpyArray((ninputs+1, nhidden), np.float)), MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, noutputs), np.float)) ] def execute0(self): """ This method does the actual execution. """ input0 = self.yet_previous_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() input1 = self.previous_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() input = np.append(input0, input1) output = self.current_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() predicted = self.MLP.train(input.astype(np.float)/255, output.astype(np.float)/255) predicted = 255*predicted.reshape((self.block[2], self.block[3], 3)) self.predicted_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]]=predicted.astype(np.uint8) def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/demo03_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time import cv2 import numpy as np import PVM_framework.PVM_Create as PVM_Create import PVM_framework.CoreUtils as CoreUtils import PVM_framework.SharedArray as SharedArray import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import os import logging import importlib import argparse class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run, cam): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True self.current_frame = None self.cam = cam def start(self): """ This will be called right before the simulation starts """ self.t_start = time.time() self.steps = 0 ret, frame = self.cam.read() self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['array'][0].shape[1], self.prop_dict['array'][0].shape[0])) self.current_frame1 = cv2.pyrDown(self.current_frame) self.current_frame2 = cv2.pyrDown(self.current_frame1) self.current_frame3 = cv2.resize(self.current_frame2, dsize=(self.prop_dict['array3'][0].shape[1], self.prop_dict['array3'][0].shape[0])) def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ self.prop_dict['array'][:] = np.roll(self.prop_dict['array'], 1, axis=0) self.prop_dict['array'][0][:] = self.current_frame[:] self.prop_dict['array1'][:] = np.roll(self.prop_dict['array1'], 1, axis=0) self.prop_dict['array1'][0][:] = self.current_frame1[:] self.prop_dict['array2'][:] = np.roll(self.prop_dict['array2'], 1, axis=0) self.prop_dict['array2'][0][:] = self.current_frame2[:] self.prop_dict['array3'][:] = np.roll(self.prop_dict['array3'], 1, axis=0) self.prop_dict['array3'][0][:] = self.current_frame3[:] def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ ret, frame = self.cam.read() self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['array'][0].shape[1], self.prop_dict['array'][0].shape[0])) self.current_frame1 = cv2.pyrDown(self.current_frame) self.current_frame2 = cv2.pyrDown(self.current_frame1) self.current_frame3 = cv2.resize(self.current_frame2, dsize=(self.prop_dict['array3'][0].shape[1], self.prop_dict['array3'][0].shape[0])) im0 = np.hstack((self.prop_dict['array'][0].view(np.ndarray), self.prop_dict['predicted_array'][0].view(np.ndarray), cv2.absdiff(self.prop_dict['predicted_array'][0].view(np.ndarray), self.prop_dict['array'][0].view(np.ndarray)))) im1 = np.hstack((self.prop_dict['array1'][0].view(np.ndarray), self.prop_dict['predicted_array1'][0].view(np.ndarray), cv2.absdiff(self.prop_dict['predicted_array1'][0].view(np.ndarray), self.prop_dict['array1'][0].view(np.ndarray)))) im1 = cv2.resize(im1, dsize=(im0.shape[1], im0.shape[0])) im2 = np.hstack((self.prop_dict['array2'][0].view(np.ndarray), self.prop_dict['predicted_array2'][0].view(np.ndarray), cv2.absdiff(self.prop_dict['predicted_array2'][0].view(np.ndarray), self.prop_dict['array2'][0].view(np.ndarray)))) im2 = cv2.resize(im2, dsize=(im0.shape[1], im0.shape[0])) im3 = np.hstack((self.prop_dict['array3'][0].view(np.ndarray), self.prop_dict['predicted_array3'][0].view(np.ndarray), cv2.absdiff(self.prop_dict['predicted_array3'][0].view(np.ndarray), self.prop_dict['array3'][0].view(np.ndarray)))) im3 = cv2.resize(im3, dsize=(im0.shape[1], im0.shape[0])) im = np.vstack((im0, im1, im2, im3)) cv2.imshow("Scale3", im) # imshow is slow, its better to call it just once key = cv2.waitKey(1) & 0xFF if key == 27 or self.steps > self.steps_to_run: self._running = False now = time.time() self.steps += 1 sps = self.steps / (now-self.t_start) print ("%3.3f steps per sec" % (sps)) + "\r", def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass def get_neighbours(x, y, sizex, sizey): result = [] if x > 0: result.append([x-1, y]) if y > 0: result.append([x, y-1]) if x < sizex-1: result.append([x+1, y]) if y < sizey-1: result.append([x, y+1]) return result def create_unit_parameters(learning_rate, momentum, block, array, predicted_array, internal_rep_size, context_block, output_block): unit_parameters = {} unit_parameters['learning_rate'] = learning_rate unit_parameters['momentum'] = momentum unit_parameters['array'] = array unit_parameters['predicted_array'] = predicted_array unit_parameters['internal_rep_size'] = internal_rep_size unit_parameters['output_block'] = output_block unit_parameters['context_blocks'] = [context_block] # block will define a sub-block of the arena that each unit will be processing # it will contain (x0, y0, width, height) unit_parameters['block'] = block return unit_parameters def generate_dict(blocks_per_dim = 8, block_size = 8): # simulation dictionary has to have: # N - SharedArray of one entity enumerating the step of the simulation # stages - integer number of processing stages # num_proc - integer number of processors that will be utilized for workers # stage0 - a list of element parameters (dictionaries) for each execution element # stage0_size - number of execution elements in the stage # other parameters are model specific. simulation_dict = PVM_Create.create_blank_dictionary() simulation_dict['stages'] = 1 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = 'PVM_models.demo03_unit' unit = importlib.import_module(simulation_dict['execution_unit_module']) internal_rep_size = 2*block_size*block_size*3/10 simulation_dict['stage0_size'] = 3*blocks_per_dim*blocks_per_dim+1 # Create the high resolution x 3 frame array array = SharedArray.SharedNumpyArray((3, block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['array'] = array predicted_array = SharedArray.SharedNumpyArray((1, block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['predicted_array'] = predicted_array # Create the medium resolution x 12 frame array array1 = SharedArray.SharedNumpyArray((4*3, block_size*blocks_per_dim/2, block_size*blocks_per_dim/2, 3), np.uint8) simulation_dict['array1'] = array1 predicted_array1 = SharedArray.SharedNumpyArray((4, block_size*blocks_per_dim/2, block_size*blocks_per_dim/2, 3), np.uint8) simulation_dict['predicted_array1'] = predicted_array1 # Create the low resolution x 48 frame array array2 = SharedArray.SharedNumpyArray((16*3, block_size*blocks_per_dim/4, block_size*blocks_per_dim/4, 3), np.uint8) simulation_dict['array2'] = array2 predicted_array2 = SharedArray.SharedNumpyArray((16, block_size*blocks_per_dim/4, block_size*blocks_per_dim/4, 3), np.uint8) simulation_dict['predicted_array2'] = predicted_array2 # Create the low resolution x 3 frame array array3 = SharedArray.SharedNumpyArray((3, block_size, block_size, 3), np.uint8) simulation_dict['array3'] = array3 predicted_array3 = SharedArray.SharedNumpyArray((1, block_size, block_size, 3), np.uint8) simulation_dict['predicted_array3'] = predicted_array3 context_block = SharedArray.SharedNumpyArray((internal_rep_size,), np.float) # Create some basic MLP parameters learning_rate = SharedArray.SharedNumpyArray((1,), np.float) learning_rate[0] = 0.01 simulation_dict['learning_rate'] = learning_rate momentum = SharedArray.SharedNumpyArray((1,), np.float) momentum[0] = 0.5 simulation_dict['momentum'] = momentum # Run loops to create and connect all the units # High spatial resolution, low temporal cover layer for x in range(blocks_per_dim): for y in range(blocks_per_dim): output_block = SharedArray.SharedNumpyArray((internal_rep_size,), np.float) block = SharedArray.SharedNumpyArray((4,), np.int) block[0] = x*block_size block[1] = y*block_size block[2] = block_size block[3] = block_size unit_parameters = create_unit_parameters(learning_rate, momentum, block, array, predicted_array, internal_rep_size, context_block, output_block) simulation_dict['stage0'].append(unit_parameters) # Medium spatial resolution, medium temporal cover layer for x in range(blocks_per_dim): for y in range(blocks_per_dim): output_block = SharedArray.SharedNumpyArray((internal_rep_size,), np.float) block = SharedArray.SharedNumpyArray((4,), np.int) block[0] = x*(block_size/2) block[1] = y*(block_size/2) block[2] = block_size/2 block[3] = block_size/2 unit_parameters = create_unit_parameters(learning_rate, momentum, block, array1, predicted_array1, internal_rep_size, context_block, output_block) simulation_dict['stage0'].append(unit_parameters) # Low spatial resolution, high temporal cover layer for x in range(blocks_per_dim): for y in range(blocks_per_dim): output_block = SharedArray.SharedNumpyArray((internal_rep_size,), np.float) block = SharedArray.SharedNumpyArray((4,), np.int) block[0] = x*(block_size/4) block[1] = y*(block_size/4) block[2] = block_size/4 block[3] = block_size/4 unit_parameters = create_unit_parameters(learning_rate, momentum, block, array2, predicted_array2, internal_rep_size, context_block, output_block) simulation_dict['stage0'].append(unit_parameters) # Each upper layer will provide its internal activations as context to the lower layer. for x in range(blocks_per_dim): for y in range(blocks_per_dim): i = x*blocks_per_dim+y simulation_dict['stage0'][i+blocks_per_dim*blocks_per_dim]['context_blocks'].append(simulation_dict['stage0'][i+2*blocks_per_dim*blocks_per_dim]['output_block']) simulation_dict['stage0'][i]['context_blocks'].append(simulation_dict['stage0'][i+blocks_per_dim*blocks_per_dim]['output_block']) surround = get_neighbours(x, y, blocks_per_dim, blocks_per_dim) surround_idx = map(lambda a: a[0]*blocks_per_dim+a[1], surround) for idx in surround_idx: simulation_dict['stage0'][i]['context_blocks'].append(simulation_dict['stage0'][idx]['output_block']) simulation_dict['stage0'][i+blocks_per_dim*blocks_per_dim]['context_blocks'].append(simulation_dict['stage0'][idx+blocks_per_dim*blocks_per_dim]['output_block']) simulation_dict['stage0'][i+2*blocks_per_dim*blocks_per_dim]['context_blocks'].append(simulation_dict['stage0'][idx+2*blocks_per_dim*blocks_per_dim]['output_block']) # This single unit will cover the entire scene and supply its internal activations as # context to everyone. for i in range(1): output_block = context_block block = SharedArray.SharedNumpyArray((4,), np.int) # each unit gets a 100x100 sub-block of the arena block[0] = 0 block[1] = 0 block[2] = block_size block[3] = block_size unit_parameters = create_unit_parameters(learning_rate, momentum, block, array3, predicted_array3, internal_rep_size, context_block, output_block) simulation_dict['stage0'].append(unit_parameters) for i in range(simulation_dict['stage0_size']): unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i]) return simulation_dict def run_demo(movie_file): """ In this demo, an image is being predicted by a set of predictive encoders looking at different aspects of the input. There is a set of future encoders digesting two 8x8 frames to predict the next one, another set taking 8 4x4 frames to predict 4 subsequent frames and another set taking 32 2x2 frames to predict 16 subsequent frames. Additionally there is one unit taking the whole image as 8x8 block whose internal representations are shared as context with all the other units. The system has feedback connections, more temporal areas feed back to more spatial areas. Also cross-like neighbourhood of lateral projections is instantiated. The simulation is synchronous, runs in single stage. """ filename = "demo03_state.p.gz" if movie_file != "": cam = cv2.VideoCapture(movie_file) else: cam = cv2.VideoCapture(-1) if not cam.isOpened(): logging.error("Either cannot read the input file or no camera found!") exit(1) if os.path.isfile(filename): state_dict = CoreUtils.load_model(filename) else: state_dict = generate_dict() manager = Manager(state_dict, 1000000, cam=cam) CoreUtils.run_model(state_dict, manager) CoreUtils.save_model(state_dict, filename) if __name__ == '__main__': logging.basicConfig(filename="demo03.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s') logging.getLogger().addHandler(logging.StreamHandler()) # Send logs to stdout parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="Movie file to load. If empty the script will try to use the camera", type=str, default="") args = parser.parse_args() run_demo(args.file) ================================================ FILE: PVM_models/demo03_unit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 2 # because there is execute0 and execute1 implemented def __init__(self, parameters): self.array = parameters['array'].view(np.ndarray) self.divider = parameters['array'].shape[0]-(2*parameters['array'].shape[0])/3 self.predicted_array = parameters['predicted_array'].view(np.ndarray) self.block = parameters['block'].view(np.ndarray) self.output_block = parameters['output_block'] self.context_blocks = parameters['context_blocks'] self.MLP = MLP.MLP(parameters["MLP_parameters"]) @staticmethod def generate_missing_parameters(parameters): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate """ ninputs = ((2*parameters['array'].shape[0])/3)*parameters['block'][2]*parameters['block'][3]*3 for block in parameters["context_blocks"]: ninputs += np.prod(block.shape) nhidden = parameters['internal_rep_size'] noutputs = (parameters['array'].shape[0]-(2*parameters['array'].shape[0])/3)*parameters['block'][2]*parameters['block'][3]*3 parameters["MLP_parameters"] = {} parameters["MLP_parameters"]['layers'] = [ {'activation': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'error': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((ninputs+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'error': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((noutputs+1), np.float) }, ] parameters["MLP_parameters"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['beta'][0] = 1.0 parameters["MLP_parameters"]['learning_rate'] = parameters['learning_rate'] parameters["MLP_parameters"]['momentum'] = parameters['momentum'] parameters["MLP_parameters"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['weights'] = [ MLP.initialize_weights(SharedArray.SharedNumpyArray((ninputs+1, nhidden), np.float)), MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, noutputs), np.float)) ] def execute0(self): """ This method does the actual execution. """ input = self.array[self.divider:, self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() for block in self.context_blocks: input = np.append(input, block.flatten()) output = self.array[0:self.divider, self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() predicted = self.MLP.train(input.astype(np.float)/255, output.astype(np.float)/255) predicted = 255*predicted.reshape((self.divider, self.block[2], self.block[3], 3)) self.predicted_array[:, self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]] = predicted.astype(np.uint8) def execute1(self): # This has to happen after a barrier so that all the data is consistent self.output_block[:] = self.MLP.get_activation(1) def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/demo04_run.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import multiprocessing as mp import time import cv2 import numpy as np import PVM_framework.PVM_Create as PVM_Create import PVM_framework.CoreUtils as CoreUtils import PVM_framework.SharedArray as SharedArray import PVM_framework.AbstractExecutionManager as AbstractExecutionManager import os import logging import importlib import argparse class Manager(AbstractExecutionManager.ExecutionManager): def __init__(self, prop_dict, steps_to_run, cam): self.prop_dict = prop_dict self.steps_to_run = steps_to_run self._running = True self.cam = cam self.steps = 0 def start(self): """ This will be called right before the simulation starts """ self.t_start = time.time() ret, frame = self.cam.read() self.yet_previous_frame1 = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) ret, frame = self.cam.read() self.yet_previous_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) ret, frame = self.cam.read() self.previous_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) ret, frame = self.cam.read() self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) def fast_action(self): """ This is the time between steps of execution Data is consistent, but keep this piece absolutely minimal """ # switch buffers self.prop_dict['yet_previous_array1'].copyto(self.yet_previous_frame1) self.prop_dict['yet_previous_array'].copyto(self.yet_previous_frame) self.prop_dict['previous_array'].copyto(self.previous_frame) self.prop_dict['current_array'].copyto(self.current_frame) pass def slow_action(self): """ This is while the workers are running. You may do a lot of work here (preferably not more than the time of execution of workers). """ ret, frame = self.cam.read() self.yet_previous_frame1 = self.yet_previous_frame self.yet_previous_frame = self.previous_frame self.previous_frame = self.current_frame self.current_frame = cv2.resize(frame, dsize=(self.prop_dict['current_array'].shape[1], self.prop_dict['current_array'].shape[0])) self.current_frame[0:30, 0:30, :] = np.random.randint(0, 255, (30, 30, 3)) cv2.putText(self.current_frame, "X", (self.steps % 50 + 80, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) cv2.imshow("Current frame", self.prop_dict['current_array'].view(np.ndarray)) cv2.imshow("Previous frame", self.prop_dict['previous_array'].view(np.ndarray)) cv2.imshow("Predicted frame", self.prop_dict['predicted_array'].view(np.ndarray)) cv2.imshow("First order error", self.prop_dict['first_order_error'].view(np.ndarray)) cv2.imshow("Second order error", self.prop_dict['second_order_error'].view(np.ndarray)) cv2.imshow("Difference", cv2.absdiff(self.prop_dict['predicted_array'].view(np.ndarray), self.prop_dict['previous_array'].view(np.ndarray))) key = cv2.waitKey(1) & 0xFF if key == 27 or self.steps > self.steps_to_run: self._running = False now = time.time() self.steps += 1 sps = self.steps / (now-self.t_start) print ("%3.3f steps per sec" % (sps)) + "\r", def running(self): """ While returning True the simulation will keep going. """ return self._running def finish(self): pass def generate_dict(): # simulation dictionary has to have: # N - SharedArray of one entity enumerating the step of the simulation # stages - integer number of processing stages # num_proc - integer number of processors that will be utilized for workers # stage0 - a list of element parameters (dictionaries) for each execution element # stage0_size - number of execution elements in the stage # other parameters are model specific. simulation_dict = PVM_Create.create_blank_dictionary() simulation_dict['stages'] = 1 simulation_dict['num_proc'] = mp.cpu_count()/2 simulation_dict['stage0'] = [] simulation_dict['execution_unit_module'] = 'PVM_models.demo04_unit' unit = importlib.import_module(simulation_dict['execution_unit_module']) blocks_per_dim = 30 block_size = 5 simulation_dict['stage0_size'] = blocks_per_dim*blocks_per_dim yet_previous_array1 = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['yet_previous_array1'] = yet_previous_array1 yet_previous_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['yet_previous_array'] = yet_previous_array previous_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['previous_array'] = previous_array current_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['current_array'] = current_array predicted_array = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['predicted_array'] = predicted_array first_order_error = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['first_order_error'] = first_order_error second_order_error = SharedArray.SharedNumpyArray((block_size*blocks_per_dim, block_size*blocks_per_dim, 3), np.uint8) simulation_dict['second_order_error'] = second_order_error learning_rate = SharedArray.SharedNumpyArray((1,), np.float) learning_rate[0] = 0.1 simulation_dict['learning_rate'] = learning_rate momentum = SharedArray.SharedNumpyArray((1,), np.float) momentum[0] = 0.5 simulation_dict['momentum'] = momentum for i in range(simulation_dict['stage0_size']): unit_parameters = {} unit_parameters['learning_rate'] = learning_rate unit_parameters['momentum'] = momentum unit_parameters['yet_previous_array1'] = yet_previous_array1 unit_parameters['yet_previous_array'] = yet_previous_array unit_parameters['previous_array'] = previous_array unit_parameters['current_array'] = current_array unit_parameters['predicted_array'] = predicted_array unit_parameters['first_order_error'] = first_order_error unit_parameters['second_order_error'] = second_order_error # block will define a sub-block of the arena that each unit will be processing # it will contain (x0, y0, width, height) unit_parameters['block'] = SharedArray.SharedNumpyArray((4,), np.int) # each unit gets a 100x100 sub-block of the arena unit_parameters['block'][0] = (i / blocks_per_dim)*block_size unit_parameters['block'][1] = (i % blocks_per_dim)*block_size unit_parameters['block'][2] = block_size unit_parameters['block'][3] = block_size simulation_dict['stage0'].append(unit_parameters) for i in range(simulation_dict['stage0_size']): unit.ExecutionUnit.generate_missing_parameters(simulation_dict['stage0'][i]) return simulation_dict def run_demo(movie_file): """ In this demo a future/predictive encoder is being instantiated to predict a camera image based on two previous frames. The encoder predicts the signal and its own error on that signal, that is additional set of units are trying to predict the magnitude of error between the prediction and the signal. In addition the hidden layer activations from the previous step of execution are used as the context block. Second order error is calculated as the error of the error prediction. Also, the learning rate of the primary signal is modulated by the magnitude of the second order error. """ if movie_file != "": cam = cv2.VideoCapture(movie_file) else: cam = cv2.VideoCapture(-1) if not cam.isOpened(): logging.error("Either cannot read the input file or no camera found!") exit(1) filename = "demo04_state.p.gz" if os.path.isfile(filename): state_dict = CoreUtils.load_model(filename) else: state_dict = generate_dict() manager = Manager(state_dict, 1000000, cam=cam) CoreUtils.run_model(state_dict, manager) CoreUtils.save_model(state_dict, filename) if __name__ == '__main__': logging.basicConfig(filename="demo04.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s') logging.getLogger().addHandler(logging.StreamHandler()) # Send logs to stdout parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="Movie file to load. If empty the script will try to use the camera", type=str, default="") args = parser.parse_args() run_demo(args.file) ================================================ FILE: PVM_models/demo04_unit.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import PVM_framework.AbstractExecutionUnit as AbstractExecutionUnit import PVM_framework.SharedArray as SharedArray import PVM_framework.MLP as MLP class ExecutionUnit(AbstractExecutionUnit.ExecutionUnit): @classmethod def execution_steps(cls): """ The method needs to have sufficient number of execute methods :return: """ return 1 # because there is only execute0 implemented def __init__(self, parameters): self.yet_previous_array1 = parameters['yet_previous_array1'].view(np.ndarray) self.yet_previous_array = parameters['yet_previous_array'].view(np.ndarray) self.previous_array = parameters['previous_array'].view(np.ndarray) self.current_array = parameters['current_array'].view(np.ndarray) self.predicted_array = parameters['predicted_array'].view(np.ndarray) self.first_order_error = parameters['first_order_error'].view(np.ndarray) self.second_order_error = parameters['second_order_error'].view(np.ndarray) self.block = parameters['block'].view(np.ndarray) self.MLP = MLP.MLP(parameters["MLP_parameters"]) @staticmethod def generate_missing_parameters(parameters): """ This method can be called to generate all the missing dictionary parameters when all the other relevant variables are known. Leave empty if there is nothing more to generate """ ninputs1 = 3*parameters['block'][2]*parameters['block'][3]*3 nhidden = ninputs1 / 5 ninputs = ninputs1+nhidden noutputs = 2*parameters['block'][2]*parameters['block'][3]*3 parameters["MLP_parameters"] = {} parameters["MLP_parameters"]['layers'] = [ {'activation': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'error': SharedArray.SharedNumpyArray((ninputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((ninputs+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'error': SharedArray.SharedNumpyArray((nhidden+1), np.float), 'delta': SharedArray.SharedNumpyArray((nhidden+1), np.float) }, {'activation': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'error': SharedArray.SharedNumpyArray((noutputs+1), np.float), 'delta': SharedArray.SharedNumpyArray((noutputs+1), np.float) }, ] parameters["MLP_parameters"]['beta'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['beta'][0] = 1.0 parameters["MLP_parameters"]['learning_rate'] = parameters['learning_rate'] parameters["MLP_parameters"]['momentum'] = parameters['momentum'] parameters["MLP_parameters"]['mse'] = SharedArray.SharedNumpyArray((1,), np.float) parameters["MLP_parameters"]['weights'] = [ MLP.initialize_weights(SharedArray.SharedNumpyArray((ninputs+1, nhidden), np.float)), MLP.initialize_weights(SharedArray.SharedNumpyArray((nhidden+1, noutputs), np.float)) ] def execute0(self): """ In this demo a future/predictive encoder is being instantiated to predict a camera image based on two previous frames. The encoder predicts the signal and its own error on that signal, that is additional set of units are trying to predict the magnitude of error between the prediction and the signal. In addition the hidden layer activations from the previous step of execution are used as the context block. Second order error is calculated as the error of the error prediction. Also, the learning rate of the primary signal is modulated by the magnitude of the second order error. """ input00 = self.yet_previous_array1[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() input0 = self.yet_previous_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() input1 = self.previous_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() input = np.concatenate((input00, input0, input1)).astype(np.float)/255 input = np.append(input, self.MLP.get_activation(1)) predicted = self.MLP.evaluate(input) error = np.zeros_like(predicted) actual_signal = self.current_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]].flatten() actual_signal = actual_signal.astype(np.float)/255 error[:actual_signal.shape[0]] = actual_signal-predicted[:actual_signal.shape[0]] error[actual_signal.shape[0]:] = np.abs(error[:actual_signal.shape[0]]) - predicted[actual_signal.shape[0]:] error[:actual_signal.shape[0]] = np.multiply(error[:actual_signal.shape[0]], np.abs(error[actual_signal.shape[0]:])) self.MLP.train2(error) predicted = 255*predicted[:actual_signal.shape[0]].reshape((self.block[2], self.block[3], 3)) self.predicted_array[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]] = predicted.astype(np.uint8) first_order_error = error[:actual_signal.shape[0]].reshape((self.block[2], self.block[3], 3)) self.first_order_error[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]] = (first_order_error*128+128).astype(np.uint8) second_order_error = error[actual_signal.shape[0]:].reshape((self.block[2], self.block[3], 3)) self.second_order_error[self.block[0]:self.block[0]+self.block[2], self.block[1]:self.block[1]+self.block[3]] = (second_order_error*128+128).astype(np.uint8) def cleanup(self): """ This needs to be implemented but may be empty if the entire state is always kept in the dictionary elements (external). If some internal state exists, here is the place to copy it back to an external variable. """ pass ================================================ FILE: PVM_models/model_zoo/classic_medium.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "1", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "green_ball", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "900000", "enable_lateral_at": "700000", "enable_readout": [ "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00001", "hidden_block_size": "7", "initial_learning_rate": "0.0002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "8", "4", "3", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "0", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "0", "readout_block_size": [ "1", "2", "4", "6", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "0", "steps": "100000000", "stereo": "0", "supervised": "1", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "1", "use_error": "1", "use_global_backprop": "0", "use_integral": "1", "use_t_minus_2_block": "0", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/model_zoo/experiment1_face.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "1", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "face", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "900000", "enable_lateral_at": "700000", "enable_readout": [ "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00001", "hidden_block_size": "7", "initial_learning_rate": "0.0002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "8", "4", "3", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "0", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "0", "readout_block_size": [ "1", "2", "4", "6", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "0", "steps": "100000000", "stereo": "0", "supervised": "1", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "1", "use_error": "1", "use_global_backprop": "0", "use_integral": "1", "use_t_minus_2_block": "0", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/model_zoo/experiment1_green_ball.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "1", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "green_ball", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "900000", "enable_lateral_at": "700000", "enable_readout": [ "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00001", "hidden_block_size": "7", "initial_learning_rate": "0.0002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "8", "4", "3", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "0", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "0", "readout_block_size": [ "1", "2", "4", "6", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "0", "steps": "100000000", "stereo": "0", "supervised": "1", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "1", "use_error": "1", "use_global_backprop": "0", "use_integral": "1", "use_t_minus_2_block": "0", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/model_zoo/experiment1_stop_sign.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "1", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "stop_sign", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "900000", "enable_lateral_at": "700000", "enable_readout": [ "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00001", "hidden_block_size": "7", "initial_learning_rate": "0.0002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "8", "4", "3", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "0", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "0", "readout_block_size": [ "1", "2", "4", "6", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "0", "steps": "100000000", "stereo": "0", "supervised": "1", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "1", "use_error": "1", "use_global_backprop": "0", "use_integral": "1", "use_t_minus_2_block": "0", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/model_zoo/experiment2.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "0", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "non_spec", "delay_each_layer_learning": "100000", "delay_final_learning_rate": "100000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "900000", "enable_lateral_at": "700000", "enable_readout": [ "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00001", "hidden_block_size": "7", "initial_learning_rate": "0.0002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "8", "4", "3", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "0", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "0", "readout_block_size": [ "1", "2", "4", "6", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "0", "steps": "100000000", "stereo": "0", "supervised": "0", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "1", "use_error": "1", "use_global_backprop": "0", "use_integral": "1", "use_t_minus_2_block": "0", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/model_zoo/very_deep.json ================================================ { "autoencoder": "0", "backpropagate_readout_error": "0", "bias_free": "0", "bias_free_readout": "0", "context_exclude_self": "0", "dataset": "green_ball", "delay_each_layer_learning": "1000", "delay_final_learning_rate": "1000000", "disable_feedback": "0", "disable_lateral": "0", "enable_feedback_at": "00000", "enable_lateral_at": "000000", "enable_readout": [ "1", "1", "1", "1", "1", "1", "1", "1", "1", "1" ], "ex_module": "PVM_models.PVM_unit_v1", "fan_in_radius": "2", "fan_in_square_size": "2", "feed_context_in_complex_layer": "0", "final_learning_rate": "0.00005", "hidden_block_size": "7", "initial_learning_rate": "0.002", "input_block_size": "5", "last_layer_context_to_all": "1", "lateral_radius": "2", "layer_shapes": [ "16", "12", "8", "8", "6", "4", "4", "4", "2", "1" ], "model_type": "small2", "momentum": "0.5", "new_name": "", "normalize_output": "1", "only_one_channel": "0", "polynomial": "1", "predict_two_steps": "1", "readout_block_size": [ "1", "2", "2", "2", "4", "4", "4", "8", "8", "16" ], "readout_depth": "1", "reverse": "0", "save_source_files": "0", "send_context_two_layers_back": "1", "steps": "100000000", "stereo": "0", "supervised": "0", "supervised_rate": "0.00001", "tau": "0.98", "unit_type": "simple", "use_derivative": "0", "use_error": "0", "use_global_backprop": "0", "use_integral": "0", "use_t_minus_2_block": "1", "version_major": "1", "version_minor": "0" } ================================================ FILE: PVM_models/process_dream_data.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.CoreUtils as CoreUtils import numpy as np import cv2 import sys import argparse def disp(winname, im): res = cv2.resize((im*255).astype(np.uint8), dsize=(200, 200)) cv2.imshow(winname, res) def normalized_dot(a, b): return np.dot(a.flatten(), b.flatten())/(np.sqrt(np.dot(a.flatten(), a.flatten()))*np.sqrt(np.dot(b.flatten(), b.flatten()))) def rotate(li, x): return li[-x % len(li):] + li[:-x % len(li)] def process_data(filename, s): """ Processes the information collected in a dream experiment from a PVM model. A sequence of frames obtained in a dream mode is compared with the original seqience of frames. All the pairwise correlations are computed and displayed in a 2d plot. A light colored diagonal feature (top-left to bottom right diagonal) corresponds with a recreated subsequence of the original sequence. :param filename: :param s: :return: """ dream_data = CoreUtils.load_model(filename) frames = len(dream_data["stage0_data"]) dot_im = np.zeros((frames, frames)) dot_im_self = np.zeros((frames, frames)) e = np.exp(1) dream_data["stage0_data"] = rotate(dream_data["stage0_data"], -s) for x in range(frames): for y in range(frames): dot_im[x, y] = (np.exp(normalized_dot(dream_data["stage0_data"][x][1], dream_data["stage1_data"][y]))-1.0)/e print "*", sys.stdout.flush() dot_im_int = (255*dot_im).astype(np.uint8) cv2.putText(dot_im_int, "Sequence shift %d" % s, (25, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.3, color=(0, 0, 0), lineType=cv2.CV_AA) # dot_im_int_self = (255*dot_im_self).astype(np.uint8) cv2.imwrite("dream_correlation_%05d.jpg" % s, dot_im_int) # cv2.imwrite("dream_self_correlation.jpg", dot_im_int) cv2.imshow("Dream correlation", dot_im_int) # cv2.imshow("Dream self correlation", dot_im_int_self) for x in range(10): cv2.waitKey(30) print "--------------------------" if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", help="Movie file to load. If empty the script will try to use the camera", type=str, default="") parser.add_argument("-s", "--shift", help="Shift from the beginning of the sequence", type=str, default="0") parser.add_argument("-r", "--range", help="Range of the files", type=str, default="") args = parser.parse_args() if args.range == "": process_data(args.file, int(args.shift)) else: for i in range(0, 124): s = i*10 ================================================ FILE: PVM_models/run_tracking_benchmark.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import matplotlib matplotlib.use('Agg') import os import logging import PVM_framework.PVM_Storage as PVM_Storage import argparse from PVM_tools.benchmark import TrackerBenchmark from PVM_models.PVM_tracker import PVMVisionTracker import glob import PVM_framework.PVM_datasets as PVM_datasets from boto.utils import get_instance_metadata if __name__ == "__main__": logging.basicConfig(filename="PVM_tracker.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(thread)d PVM_tracker : %(message)s ') desc = "" have_display = True try: from Tkinter import Tk Tk() except: have_display = False print "No display detected, turning off visualizations." parser = argparse.ArgumentParser(description=desc) parser.add_argument("-c", "--channel", type=str, default="default", help="Channel") parser.add_argument("-k", "--cores", type=str, default="4", help="NUmber of cores to execute on") parser.add_argument("-d", "--directory", type=str, default="DARPA/DATA/TrackerBenchmark", help="Directory indicating the dataset to un on") parser.add_argument("-f", "--filename", type=str, default="", help="Filename to run on") parser.add_argument("-r", "--remote", default="", help="Filename in the could") parser.add_argument("-e", "--execute", help="Actually run the trackers", action="store_true") parser.add_argument("-p", "--prefix", type=str, default="", help="Output directory prefix") parser.add_argument("-o", "--output", type=str, default="", help="Directory to either store or load from") parser.add_argument("-s", "--set", help="Name of the dataset", type=str, default="green_ball") parser.add_argument("-n", "--no_gt", help="Dont display ground truth", action="store_true") parser.add_argument("-R", "--resolution", help="Resolution ", type=str, default="96") parser.add_argument("-S", "--steps_per_frame", help="How many steps of the PVM before an estimate is made ", type=str, default="1") parser.add_argument("-T0", "--tracker_PVM", help="Use PVM tracker ", action="store_true") parser.add_argument("-T1", "--tracker_null", help="Use null tracker ", action="store_true") parser.add_argument("-T2", "--tracker_center", help="Use center tracker ", action="store_true") parser.add_argument("-T3", "--tracker_HS", help="Use HS color histogram tracker ", action="store_true") parser.add_argument("-T4", "--tracker_UV", help="Use UV color histogram tracker ", action="store_true") parser.add_argument("-T5", "--tracker_TLD", help="Use opentld tracker ", action="store_true") parser.add_argument("-T6", "--tracker_CMT", help="Use CMT tracker ", action="store_true") parser.add_argument("-T7", "--tracker_STRUCK", help="Use STRUCK tracker ", action="store_true") args = parser.parse_args() remove_files = True ts = PVM_Storage.Storage() try: meta = get_instance_metadata(timeout=2, num_retries=2) except: meta = {} if meta == {}: remove_files = False if args.output == "": output_dir = os.path.expanduser("~/benchmark_results/")+args.prefix output_dir = os.path.normpath(output_dir) else: output_dir = os.path.normpath(args.output) if args.set == "all": test_files_with_channel = [] for ds in ("stop_sign", "green_ball", "face"): test_files_with_channel.extend(PVM_datasets.sets["%s_training" % ds]) else: test_files_with_channel = PVM_datasets.sets["%s_testing" % args.set] test_files = [] for (file, channel) in test_files_with_channel: test_files.append(file) if args.execute: trackers = [] if args.tracker_null: from other_trackers.null_vision_tracker import NullVisionTracker trackers.append(NullVisionTracker()) if args.tracker_center: from other_trackers.center_vision_tracker import CenterVisionTracker trackers.append(CenterVisionTracker()) if args.tracker_CMT: from other_trackers.cmt_vision_tracker import CMTVisionTracker trackers.append(CMTVisionTracker()) if args.tracker_TLD: from other_trackers.tld_vision_tracker import TLDVisionTracker trackers.append(TLDVisionTracker()) if args.tracker_STRUCK: from other_trackers.struck_tracker import StruckTracker trackers.append(StruckTracker()) if args.tracker_HS: from other_trackers.color_vision_tracker import HSHistogramBackprojectionTracker trackers.append(HSHistogramBackprojectionTracker()) if args.tracker_UV: from other_trackers.color_vision_tracker import UVHistogramBackprojectionTracker trackers.append(UVHistogramBackprojectionTracker()) if args.tracker_PVM: PVM_tracker = PVMVisionTracker(filename=args.filename, remote_filename=args.remote, cores=args.cores, storage=ts, steps_per_frame=int(args.steps_per_frame)) trackers.append(PVM_tracker) output_dir = os.path.expanduser("~/benchmark_results/")+args.prefix+"_"+PVM_tracker.prop_dict["name"]+"_"+str(PVM_tracker.prop_dict["N"][0]) output_dir = os.path.normpath(output_dir) TB = TrackerBenchmark(output_dir=output_dir, resolution=(int(args.resolution), int(args.resolution)), have_display=have_display, draw_gt=not args.no_gt, perturb_gt=0.4, storage=ts) TB.run(test_files, trackers, ts, channel="default", remove_downloads=remove_files) TB.save_results() else: TB = TrackerBenchmark(output_dir=output_dir, resolution=(int(args.resolution), int(args.resolution)), have_display=have_display, draw_gt=not args.no_gt, perturb_gt=0.4, storage=ts) if args.tracker_PVM: PVM_tracker.finish() TB.get_summary() TB.plot_all() benchmark_dir = output_dir base_dir = os.path.basename(os.path.normpath(benchmark_dir)) if args.tracker_PVM: to_folder = "PVM_benchmark/%s_%s_%s/Benchmark_%09d_%s/" % (PVM_tracker.prop_dict['timestamp'], PVM_tracker.prop_dict['name'], PVM_tracker.prop_dict['hash'], PVM_tracker.prop_dict["N"], base_dir) logging.info("Uploading benchmark result to Storage from %s to %s" % (os.path.normpath(benchmark_dir), to_folder)) for filename in glob.glob(os.path.normpath(benchmark_dir) + "/*"): if os.path.isfile(filename): logging.info("Inspecting file %s " % filename) ts.put(from_path=filename, to_folder=to_folder, overwrite=True) logging.info("Done, Uploading benchmark result to Amazon") ================================================ FILE: PVM_tests/test_MLP.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.MLP as MLP_module import numpy as np def test_perceptron01(): X = np.array([[0, 1], [1, 1], [1, 0], [0, 0]]) Y = np.array([[1, 0], [0, 0], [0, 1], [1, 1]]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 5, 3]) np.random.seed(seed=509) parameters['weights'] = MLP_module.get_weights(parameters['layers']) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([0.1]) parameters['momentum'] = np.array([0.5]) parameters['mse'] = np.array([0.0]) M = MLP_module.MLP(parameters) for n in xrange(30000): i = np.random.randint(low=0, high=4) M.train(X[i], Y[i]) for i in range(4): O = M.evaluate(X[i]) assert(np.allclose(Y[i], O, atol=0.03)) def test_perceptron02(): X = np.array([[0, 1], [1, 1], [1, 0], [0, 0]]) Y = np.array([[1], [0], [1], [0]]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 3, 2]) np.random.seed(seed=509) parameters['weights'] = MLP_module.get_weights(parameters['layers']) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([0.25]) parameters['momentum'] = np.array([0.5]) parameters['mse'] = np.array([0.0]) M = MLP_module.MLP(parameters) for n in xrange(30000): i = np.random.randint(low=0, high=4) M.train(X[i], Y[i]) for i in range(4): O = M.evaluate(X[i]) assert(np.allclose(Y[i], O, atol=0.03)) def test_perceptron_two_layers(): X = np.array([[0, 1], [1, 1], [1, 0], [0, 0]]) Y = np.array([[1], [1], [1], [0]]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 2]) np.random.seed(seed=509) parameters['weights'] = MLP_module.get_weights(parameters['layers']) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([0.25]) parameters['momentum'] = np.array([0.5]) parameters['mse'] = np.array([0.0]) M = MLP_module.MLP(parameters) for n in xrange(30000): i = np.random.randint(low=0, high=4) M.train(X[i], Y[i]) for i in range(4): O = M.evaluate(X[i]) assert(np.allclose(Y[i], O, atol=0.03)) def test_MLPN_eval00(): """ Simple test checking whether the MLP evaluates properly with zero weights """ x = np.array([1.0, 0.0]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 5, 5]) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([1.0]) parameters['momentum'] = np.array([0.0]) parameters['mse'] = np.array([0.0]) parameters['weights'] = [np.array([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]).astype(np.float)*100, np.array([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]).astype(np.float)*100 ] M = MLP_module.MLP(parameters) out = M.evaluate(x) assert np.allclose(out, np.array([0.5, 0.5, 0.5, 0.5])) hidden = M.get_activation(1) assert np.allclose(hidden, np.array([0.5, 0.5, 0.5, 0.5])) def test_MLPN_eval01(): """ Simple test checking whether the MLP evaluates properly on a specific case """ x3 = np.array([0.0, 1.0]) x4 = np.array([1.0, 0.0]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 6, 5]) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([1.0]) parameters['momentum'] = np.array([0.0]) parameters['mse'] = np.array([0.0]) parameters['polynomial'] = True parameters['weights'] = [np.array([[1, -1, 1, -1], [1, -1, -1, 1], [0, 0, 0, 0]]).astype(np.float)*100, np.array([[1, 0, 0, 0], [0, 2, 0, 0], [0, 0, 1, 0], [0, 0, -1, 2], [0, 0, 0, 0], [0, -1, 0, -1], ]).astype(np.float)*100 ] M = MLP_module.MLP(parameters) out = M.evaluate(x4) assert (out[0] > 0.99) assert (out[1] < 0.01) assert (out[2] > 0.99) assert (out[3] < 0.01) hidden = M.get_activation(1) assert (hidden[0] > 0.99) assert (hidden[1] < 0.01) assert (hidden[2] > 0.99) assert (hidden[3] < 0.01) out = M.evaluate(x3) assert (out[0] > 0.99) assert (out[1] < 0.01) assert (out[2] < 0.01) assert (out[3] > 0.99) hidden = M.get_activation(1) assert (hidden[0] > 0.99) assert (hidden[1] < 0.01) assert (hidden[2] < 0.01) assert (hidden[3] > 0.99) def test_MLPN_xor(): """ Simple test checking whether a 3 layer MLP is able to learn XOR """ x1 = np.array([0.0, 0.0]) x2 = np.array([1.0, 1.0]) x3 = np.array([0.0, 1.0]) x4 = np.array([1.0, 0.0]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 15, 3]) np.random.seed(seed=509) parameters['weights'] = MLP_module.get_weights(parameters['layers']) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([0.1]) parameters['momentum'] = np.array([0.5]) parameters['mse'] = np.array([0.0]) M = MLP_module.MLP(parameters) for n in xrange(30000): M.train(x1, np.array([1.0, 0.0])) M.train(x2, np.array([1.0, 0.0])) M.train(x3, np.array([0.0, 1.0])) M.train(x4, np.array([0.0, 1.0])) o1 = M.evaluate(x1) o2 = M.evaluate(x2) o3 = M.evaluate(x3) o4 = M.evaluate(x4) assert o1[0] > 0.8 assert o1[1] < 0.2 assert o2[0] > 0.8 assert o2[1] < 0.2 assert o3[0] < 0.2 assert o3[1] > 0.8 assert o4[0] < 0.2 assert o4[1] > 0.8 def test_MLPN_xor_poly(): """ Simple test checking whether a 3 layer MLP is able to learn XOR """ x1 = np.array([0.0, 0.0]) x2 = np.array([1.0, 1.0]) x3 = np.array([0.0, 1.0]) x4 = np.array([1.0, 0.0]) parameters = {} parameters['layers'] = MLP_module.get_layers([3, 15, 3]) np.random.seed(seed=509) parameters['weights'] = MLP_module.get_weights(parameters['layers']) parameters['beta'] = np.array([1.0]) parameters['learning_rate'] = np.array([0.1]) parameters['momentum'] = np.array([0.5]) parameters['mse'] = np.array([0.0]) parameters['polynomial'] = True M = MLP_module.MLP(parameters) for n in xrange(30000): M.train(x1, np.array([1.0, 0.0])) M.train(x2, np.array([1.0, 0.0])) M.train(x3, np.array([0.0, 1.0])) M.train(x4, np.array([0.0, 1.0])) o1 = M.evaluate(x1) o2 = M.evaluate(x2) o3 = M.evaluate(x3) o4 = M.evaluate(x4) assert o1[0] > 0.8 assert o1[1] < 0.2 assert o2[0] > 0.8 assert o2[1] < 0.2 assert o3[0] < 0.2 assert o3[1] > 0.8 assert o4[0] < 0.2 assert o4[1] > 0.8 if __name__ == '__main__': test_MLPN_eval00() test_MLPN_eval01() test_MLPN_xor() test_MLPN_xor_poly() test_perceptron_two_layers() ================================================ FILE: PVM_tests/test_SharedArray.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import pytest import PVM_framework.SharedArray as SharedArray import numpy as np import cPickle import gzip import multiprocessing def save(pobject, filename, protocol = -1): """Save an object to a compressed disk file. Works well with huge objects. """ pfile = gzip.GzipFile(filename, 'wb') cPickle.dump(pobject, pfile, protocol) pfile.close() def load(filename): """Loads a compressed object from disk """ pfile = gzip.GzipFile(filename, 'rb') prop_dict = cPickle.load(pfile) pfile.close() return prop_dict def test_pickling_array(tmpdir): """ Pickles and unpickles a shared array """ A = SharedArray.SharedNumpyArray((10, 10), np.float) A[1, 1] = 15 save(A, str(tmpdir)+"/test.p.gz") B = load(str(tmpdir)+"/test.p.gz") assert(B[1, 1] == 15) def test_pickling_darray(tmpdir): """ Pickles and unpickles a double buffered shared array """ parity = SharedArray.SharedNumpyArray((1,), np.uint32) A = SharedArray.DoubleBufferedSharedNumpyArray((10, 10), np.float, parity) A[1, 1] = 15 parity[0] = 1 A[1, 1] = 12 save({"array": A, "parity": parity}, str(tmpdir)+"/test.p.gz") ddict = load(str(tmpdir)+"/test.p.gz") B = ddict["array"] parity = ddict["parity"] assert(B[1, 1] == 12) parity[0] = 0 assert(B[1, 1] == 15) def worker_test_sharednes_of_an_array(Array, A): """ Worker code spawned in a separate process to test if the shared array is in fact shared. """ Array[5, 5] = 10 return def test_sharedness_of_an_array(): """ Test if the array is actually shared between two processes. """ for dtype in (np.float, np.float16, np.float32, np.float64, np.float128, np.uint8, np.uint16, np.uint32, np.uint64, np.int8, np.int16, np.int32, np.int64, np.complex, np.complex64): Array = SharedArray.SharedNumpyArray((10, 10), dtype) p = multiprocessing.Process(target=worker_test_sharednes_of_an_array, args=(Array, None)) p.start() p.join() assert(Array[5, 5] == 10) def worker_test_sharedness_of_a_darray(Array, Parity): """ Worker code spawned in a separate process to test if the double buffered shared array is in fact shared. """ Array[5, 5] = 10 Parity[0] = 1 Array[5, 5] = 15 return def test_sharedness_of_a_darray(): """ Test if the double buffered array is actually shared between two processes. """ parity = SharedArray.SharedNumpyArray((1,), np.uint32) Array = SharedArray.DoubleBufferedSharedNumpyArray((10, 10), np.float, parity) p = multiprocessing.Process(target=worker_test_sharedness_of_a_darray, args=(Array, parity)) p.start() p.join() assert(Array[5, 5] == 15) parity[0] = 0 assert(Array[5, 5] == 10) def worker_test_IPC_attachability(): Array=SharedArray.SharedNumpyArray((10, 10), np.float, tag="abcdefgh312", create=False) Array[5, 5] = 10 return def test_IPC_attachability(): """ Test if the array is actually shared between two processes. """ Array=SharedArray.SharedNumpyArray((10, 10), np.float, tag="abcdefgh312", create=True) p = multiprocessing.Process(target=worker_test_IPC_attachability, args=()) p.start() p.join() assert(Array[5, 5] == 10) if __name__ == "__main__": manual_test = True if manual_test: test_pickling_array() test_pickling_darray() test_sharedness_of_an_array() test_sharedness_of_a_darray() test_IPC_attachability() else: pytest.main('-s %s -n0' % __file__) ================================================ FILE: PVM_tests/test_SyncUtils.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import pytest import PVM_framework.SharedArray as SharedArray import numpy as np import multiprocessing test_spinlocks = False try: import PVM_framework.SyncUtils as SyncUtils test_spinlocks = True except: import PVM_framework.SyncUtils_python as SyncUtils import py def ver_to_float(ver): nums = ver.split('.') result = 0.0 base = 1.0 for num in nums: result += float(num)*base base *= 0.001 return result def worker_test_barrier(barrier, array, proc_id): """ Worker code, spawn in many copies as separate processes """ barrier.worker_barrier() condition = True while condition: array[proc_id] += 1 condition = barrier.worker_barrier() def parrent_test_barrier(UseSpinLock=False): """ Parent (controller) process, instantiates the shared memory and synchronization objects and spawns worker processes. Verifies that the data consistency is assured as the code executes. """ test_barrier = SyncUtils.Barrier(4, timeout=0.001, UseSpinLock=UseSpinLock) shared_data = SharedArray.SharedNumpyArray((4,), np.float) procs = [] for i in xrange(4): proc = multiprocessing.Process(target=worker_test_barrier, args=(test_barrier, shared_data, i)) procs.append(proc) proc.start() test_barrier.parent_barrier() test_barrier.resume_workers() for i in xrange(4): test_barrier.parent_barrier() assert (sum(shared_data) == (i + 1) * 4) test_barrier.resume_workers() test_barrier.parent_barrier(quit_workers=True) for p in procs: p.join() def test_barrier(): """ Test barrier with regular locks provided by the operating system. """ parrent_test_barrier(UseSpinLock=False) def test_barrier_spinlocks(): """ Test barrier with spin locks implemented using atomic operation on a shared array object (busy wait). Use spinlocks for highly parallel applications (>10 worker processes) where synchronization lag needs to be minimal even the expense of some busy wait. """ if (not test_spinlocks): py.test.skip("gcc > 4.6 is nescessary for this test") parrent_test_barrier(UseSpinLock=True) if __name__ == "__main__": manual_test = True if manual_test: test_barrier() test_barrier_spinlocks() else: pytest.main('-s %s -n0' % __file__) ================================================ FILE: PVM_tests/test_bounding_region.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_tools.bounding_region as bounding_region import numpy as np import cv2 def test_initialization(): """ Test if the bounding regions initialized with a box and a corresponding contour are exactly the same in every possible aspect """ shape = (100, 100, 3) b = bounding_region.BoundingRegion(shape, box=np.array([10, 20, 15, 45])) c = bounding_region.BoundingRegion(shape, contour=np.array([[[10, 20]], [[25, 20]], [[25, 65]], [[10, 65]]])) assert(np.allclose(b.get_box_pixels(), c.get_box_pixels())) assert(np.allclose(b.get_contour_pixels(), c.get_contour_pixels())) assert(np.allclose(b.get_contour_relative(), c.get_contour_relative())) assert(b.get_box_center_pixels() == c.get_box_center_pixels()) assert(b.get_box_center_relative() == c.get_box_center_relative()) assert(b.get_centroid_pixels() == c.get_centroid_pixels()) assert(b.get_centroid_relative() == c.get_centroid_relative()) assert(b.get_area_pixels() == c.get_area_pixels()) assert(b.get_area_relative() == c.get_area_relative()) assert(b.get_perimeter_pixels() == c.get_perimeter_pixels()) def test_fundamentals(): """ Tests basic funtionality to identify if the behavior of the class corresponds to the expected given the inializing data. """ shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) # Initilized with a box b = bounding_region.BoundingRegion(shape, box=np.array([10, 20, 15, 45])) b.draw_box(image, (255, 255, 0)) assert(np.allclose(image[20, 10], np.array([255, 255, 0]))) assert(np.allclose(image[20, 25], np.array([255, 255, 0]))) assert(np.allclose(image[65, 25], np.array([255, 255, 0]))) assert(np.allclose(image[65, 10], np.array([255, 255, 0]))) # Initilized with a contour image = np.zeros(shape, dtype=np.uint8) b = bounding_region.BoundingRegion(shape, contour=np.array([[[10, 20]], [[25, 20]], [[25, 65]], [[10, 65]]])) b.draw_box(image, (255, 255, 0)) assert(np.allclose(image[20, 10], np.array([255, 255, 0]))) assert(np.allclose(image[20, 25], np.array([255, 255, 0]))) assert(np.allclose(image[65, 25], np.array([255, 255, 0]))) assert(np.allclose(image[65, 10], np.array([255, 255, 0]))) def test_area(): """ Tests if the contour being drawn on the picture has the same area as the one used to initialize the class. """ shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) # Initilized with a box b = bounding_region.BoundingRegion(shape, box=np.array([10, 20, 15, 45])) b.draw_box(image, (255, 255, 255), thickness=-1) imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) contours, hierarchy = cv2.findContours(imgray, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_SIMPLE) assert(len(contours) == 1) area = cv2.contourArea(contours[0]) assert(area == b.get_area_pixels()) # Initilized with a contour image = np.zeros(shape, dtype=np.uint8) b = bounding_region.BoundingRegion(shape, contour=np.array([[[10, 20]], [[25, 20]], [[25, 65]], [[10, 65]]])) b.draw_box(image, (255, 255, 255), thickness=-1) imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) contours, hierarchy = cv2.findContours(imgray, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_SIMPLE) assert(len(contours) == 1) area = cv2.contourArea(contours[0]) assert(area == b.get_area_pixels()) def test_scaling(): """ Tests if the scaling functionality result in correct behavior """ shape = (100, 100, 3) # Initilized with a box image = np.zeros(shape) b = bounding_region.BoundingRegion(shape, box=np.array([40, 40, 20, 20])) b.scale(0.5) b.draw_box(image, (255, 255, 0)) assert(np.allclose(image[45, 45], np.array([255, 255, 0]))) assert(np.allclose(image[45, 55], np.array([255, 255, 0]))) assert(np.allclose(image[55, 55], np.array([255, 255, 0]))) assert(np.allclose(image[55, 45], np.array([255, 255, 0]))) b.scale(2.0) b.draw_box(image, (0, 255, 0)) assert(np.allclose(image[40, 40], np.array([0, 255, 0]))) assert(np.allclose(image[40, 60], np.array([0, 255, 0]))) assert(np.allclose(image[60, 40], np.array([0, 255, 0]))) assert(np.allclose(image[60, 60], np.array([0, 255, 0]))) b.scale(2.0) b.draw_box(image, (0, 255, 255)) assert(np.allclose(image[30, 30], np.array([0, 255, 255]))) assert(np.allclose(image[30, 70], np.array([0, 255, 255]))) assert(np.allclose(image[70, 30], np.array([0, 255, 255]))) assert(np.allclose(image[70, 70], np.array([0, 255, 255]))) image = np.zeros(shape) # Initilized with a contour b = bounding_region.BoundingRegion(shape, contour=np.array([[[40, 40]], [[40, 60]], [[60, 60]], [[60, 40]]])) b.scale(0.5) b.draw_box(image, (255, 255, 0)) assert(np.allclose(image[45, 45], np.array([255, 255, 0]))) assert(np.allclose(image[45, 55], np.array([255, 255, 0]))) assert(np.allclose(image[55, 55], np.array([255, 255, 0]))) assert(np.allclose(image[55, 45], np.array([255, 255, 0]))) b.scale(2.0) b.draw_box(image, (0, 255, 0)) assert(np.allclose(image[40, 40], np.array([0, 255, 0]))) assert(np.allclose(image[40, 60], np.array([0, 255, 0]))) assert(np.allclose(image[60, 40], np.array([0, 255, 0]))) assert(np.allclose(image[60, 60], np.array([0, 255, 0]))) b.scale(2.0) b.draw_box(image, (0, 255, 255)) assert(np.allclose(image[30, 30], np.array([0, 255, 255]))) assert(np.allclose(image[30, 70], np.array([0, 255, 255]))) assert(np.allclose(image[70, 30], np.array([0, 255, 255]))) assert(np.allclose(image[70, 70], np.array([0, 255, 255]))) def test_box_intersection(): """ Simple static test to verify the intersection is indeed an intersection """ shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) # Initilized with a box b1 = bounding_region.BoundingRegion(image_shape=shape, box=np.array([10, 20, 15, 45])) b2 = bounding_region.BoundingRegion(image_shape=shape, box=np.array([20, 60, 15, 25])) mask1 = b1.get_mask() mask2 = b2.get_mask() b3 = b1.get_box_intersection(b2) b3.set_image_shape(shape=shape) mask3 = b3.get_mask() mask4 = cv2.bitwise_and(mask1, mask2) assert(np.allclose(mask3, mask4)) def test_box_intersection_randomized(): """ Deeper test runs 1000 times to verify that the intersection is right each time. """ for i in xrange(1000): shape = (100, 100, 3) # Initilized with a box box1 = np.random.randint(low=10, high=30, size=(4, 1)) box2 = np.random.randint(low=10, high=30, size=(4, 1)) b1 = bounding_region.BoundingRegion(image_shape=shape, box=box1) b2 = bounding_region.BoundingRegion(image_shape=shape, box=box2) mask1 = b1.get_mask() mask2 = b2.get_mask() b3 = b1.get_box_intersection(b2) b3.set_image_shape(shape=shape) mask3 = b3.get_mask() mask4 = cv2.bitwise_and(mask1, mask2) assert(np.allclose(mask3, mask4)) def test_mask_randomized(): """ Tests that the mask generation from the box work right. """ for i in xrange(1000): shape = (100, 100) image = np.zeros(shape, dtype=np.uint8) # Initilized with a box box1 = np.random.randint(low=10, high=30, size=(4, 1)) cv2.rectangle(image, (box1[0], box1[1]), (box1[0]+box1[2], box1[1]+box1[3]), color=255, thickness=-1) b1 = bounding_region.BoundingRegion(image_shape=shape, box=box1) mask1 = b1.get_mask() assert (np.allclose(image, mask1)) if __name__ == "__main__": test_initialization() test_fundamentals() test_area() test_scaling() ================================================ FILE: PVM_tests/test_create.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.PVM_Create as PVM_Create def test_PVM_crate(): A0 = PVM_Create.get_fan_in([4, 4], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2) assert A0 == [(8, 8), (8, 9), (9, 8), (9, 9)] A0 = PVM_Create.get_fan_in([4, 4], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2, radius=1) assert A0 == [(8, 8), (8, 9), (9, 8), (9, 9)] A0 = PVM_Create.get_fan_in([4, 4], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3) assert A0 == [(7, 7), (7, 8), (7, 9), (8, 7), (8, 8), (8, 9), (9, 7), (9, 8), (9, 9)] A0 = PVM_Create.get_fan_in([4, 4], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3, radius=1) assert A0 == [(7, 8), (8, 7), (8, 8), (8, 9), (9, 8)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2) assert A0 == [(0, 0), (0, 1), (1, 0), (1, 1)] A0 = PVM_Create.get_fan_in([7, 7], dim_x_l=16, dim_y_l=16, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2) assert A0 == [(14, 14), (14, 15), (15, 14), (15, 15)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=20, dim_y_l=20, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2) assert A0 == [(0, 0), (0, 1), (1, 0), (1, 1)] A0 = PVM_Create.get_fan_in([7, 7], dim_x_l=20, dim_y_l=20, dim_x_u=8, dim_y_u=8, block_x=2, block_y=2) assert A0 == [(18, 18), (18, 19), (19, 18), (19, 19)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=24, dim_y_l=24, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3) assert A0 == [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1), (2, 2)] A0 = PVM_Create.get_fan_in([7, 7], dim_x_l=24, dim_y_l=24, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3) assert A0 == [(21, 21), (21, 22), (21, 23), (22, 21), (22, 22), (22, 23), (23, 21), (23, 22), (23, 23)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=20, dim_y_l=20, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3) assert A0 == [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1), (2, 2)] A0 = PVM_Create.get_fan_in([7, 7], dim_x_l=20, dim_y_l=20, dim_x_u=8, dim_y_u=8, block_x=3, block_y=3) assert A0 == [(17, 17), (17, 18), (17, 19), (18, 17), (18, 18), (18, 19), (19, 17), (19, 18), (19, 19)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=20, dim_y_l=20, dim_x_u=20, dim_y_u=20, block_x=3, block_y=3) assert A0 == [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1), (2, 2)] A0 = PVM_Create.get_fan_in([19, 19], dim_x_l=20, dim_y_l=20, dim_x_u=20, dim_y_u=20, block_x=3, block_y=3) assert A0 == [(17, 17), (17, 18), (17, 19), (18, 17), (18, 18), (18, 19), (19, 17), (19, 18), (19, 19)] A0 = PVM_Create.get_fan_in([18, 18], dim_x_l=20, dim_y_l=20, dim_x_u=20, dim_y_u=20, block_x=3, block_y=3) assert A0 == [(16, 16), (16, 17), (16, 18), (17, 16), (17, 17), (17, 18), (18, 16), (18, 17), (18, 18)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=5, dim_y_l=5, dim_x_u=1, dim_y_u=1, block_x=3, block_y=3) assert A0 == [(1, 1), (1, 2), (1, 3), (2, 1), (2, 2), (2, 3), (3, 1), (3, 2), (3, 3)] A0 = PVM_Create.get_fan_in([0, 0], dim_x_l=2, dim_y_l=2, dim_x_u=1, dim_y_u=1, block_x=2, block_y=2) assert A0 == [(0, 0), (0, 1), (1, 0), (1, 1)] A0 = PVM_Create.get_fan_in([5, 5], dim_x_l=10, dim_y_l=10, dim_x_u=10, dim_y_u=10, block_x=3, block_y=3, radius=3) assert A0 == [(3, 3), (3, 4), (3, 5), (4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)] A0 = PVM_Create.get_fan_in([5, 5], dim_x_l=10, dim_y_l=10, dim_x_u=10, dim_y_u=10, block_x=3, block_y=3, radius=1) assert A0 == [(3, 4), (4, 3), (4, 4), (4, 5), (5, 4)] A0 = PVM_Create.get_fan_in([5, 5], dim_x_l=10, dim_y_l=10, dim_x_u=10, dim_y_u=10, block_x=2, block_y=2, radius=1) assert A0 == [(4, 4), (4, 5), (5, 4), (5, 5)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=1, exclude_self=False) assert A0 == [(4, 5), (5, 4), (5, 5), (5, 6), (6, 5)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=1.5, exclude_self=False) assert A0 == [(4, 4), (4, 5), (4, 6), (5, 4), (5, 5), (5, 6), (6, 4), (6, 5), (6, 6)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=2, exclude_self=False) assert A0 == [(3, 5), (4, 4), (4, 5), (4, 6), (5, 3), (5, 4), (5, 5), (5, 6), (5, 7), (6, 4), (6, 5), (6, 6), (7, 5)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=1, exclude_self=True) assert A0 == [(4, 5), (5, 4), (5, 6), (6, 5)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=1.5, exclude_self=True) assert A0 == [(4, 4), (4, 5), (4, 6), (5, 4), (5, 6), (6, 4), (6, 5), (6, 6)] A0 = PVM_Create.get_surround([5, 5], dim_x=10, dim_y=10, radius=2, exclude_self=True) assert A0 == [(3, 5), (4, 4), (4, 5), (4, 6), (5, 3), (5, 4), (5, 6), (5, 7), (6, 4), (6, 5), (6, 6), (7, 5)] A0 = PVM_Create.get_surround([0, 0], dim_x=10, dim_y=10, radius=2, exclude_self=False) assert A0 == [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (2, 0)] if __name__ == "__main__": test_PVM_crate() ================================================ FILE: PVM_tests/test_fast_routines.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_framework.fast_routines as fast_routines import numpy as np import time def test_derivative_dot(): print "------ derivative_dot ------" vector = np.random.randn(5000) vector2 = np.ones((5000,)) result_cython = np.zeros((5000,)) result_c = np.zeros((5000,)) t_start = time.time() fast_routines.derivative_dot_cython(vector, vector2, result_cython) cython_time=time.time()-t_start print "Cython implementation %f" % (cython_time) t_start = time.time() fast_routines.derivative_dot_c(vector, vector2, result_c) c_time=time.time()-t_start print "C implementation %f " % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) def test_dot_transpose(): print "------ dot_transpose ------" vector = np.random.randn(30000) matrix = np.random.randn(5000, 30000) multiplier=np.ones((5000,)) result_cython=np.zeros((5000,)) result_c=np.zeros((5000,)) t_start = time.time() fast_routines.dot_transpose_mul_cython(multiplier, vector, matrix, result_cython) cython_time=time.time()-t_start print "Cython implementation %f" % (cython_time) t_start = time.time() fast_routines.dot_transpose_mul_c(multiplier, vector, matrix, result_c) c_time=time.time()-t_start print "C implementation %f " % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) def test_dot_transpose_simple(): print "------ dot_transpose_simple ------" vector = np.random.randn(30000) matrix = np.random.randn(5000, 30000) result_cython=np.zeros((5000,)) result_c=np.zeros((5000,)) t_start = time.time() fast_routines.dot_transpose_cython(vector, matrix, result_cython) cython_time=time.time()-t_start print "Cython implementation %f" % (cython_time) t_start = time.time() fast_routines.dot_transpose_c(vector, matrix, result_c) c_time=time.time()-t_start print "C implementation %f " % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) def test_generalized_outer(): print "------ generalized outer ------" d1=4000 d2=5000 vector1 = np.random.randn(d1) vector2 = np.random.randn(d2) matrix = np.random.randn(d1, d2) result_cython = np.zeros((d1, d2)) result_c = np.zeros((d1, d2)) t_start = time.time() fast_routines.generalized_outer_cython(1.0, vector1, vector2, 0.5, matrix, result_cython) cython_time = time.time()-t_start print "Cython implementation %f" % cython_time t_start = time.time() fast_routines.generalized_outer_c(1.0, vector1, vector2, 0.5, matrix, result_c) c_time=time.time()-t_start print "C implementation %f" % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) def dot_sigmoid(append_bias): print "------ dot_sigmoid ------" d1=500 d2=400 vector1 = np.random.randn(d1) matrix = np.random.randn(d1+append_bias, d2) result_cython = np.zeros((d2)) result_c = np.zeros((d2)) t_start = time.time() fast_routines.dot_sigmoid_cython(vector1, matrix, 1.0, result_cython, append_bias) cython_time = time.time()-t_start print "Cython implementation %f" % cython_time t_start = time.time() fast_routines.dot_sigmoid_c(vector1, matrix, 1.0, result_c, append_bias) c_time=time.time()-t_start print "C implementation %f" % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) def test_dot_sigmoid(): dot_sigmoid(0) dot_sigmoid(1) def test_dot_add(): d1=500 d2=400 vector1 = np.random.randn(d1) matrix = np.random.randn(d1, d2) result_c = np.zeros((d2)) t_start = time.time() result_cython=np.dot(matrix.T, vector1) cython_time = time.time()-t_start print "Cython implementation %f" % cython_time t_start = time.time() fast_routines.dot_add_c(vector1, matrix, result_c, 0) c_time=time.time()-t_start print "C implementation %f" % (c_time) print "Acceleration %f" % (cython_time/c_time) assert(np.allclose(result_cython, result_c)) if __name__=="__main__": test_dot_transpose() test_generalized_outer() test_dot_sigmoid() ================================================ FILE: PVM_tests/test_labeled_movie.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import PVM_tools.labeled_movie as labeled_movie import numpy as np def test_basic_channels_1(): img_left = np.zeros((100, 100, 3)) img_right = np.zeros((100, 100, 3))+255 F = labeled_movie.LabeledMovieFrame(internal_storage_method="jpg") F.create_channel("left_image") F.set_default_channel("left_image") F.set_image(img_left) # Now left_image is the default channel F.create_channel("right_image") F.set_image(img_right, channel="right_image") assert(np.allclose(F.get_image(), img_left)) assert(np.allclose(F.get_image(channel="left_image"), img_left)) assert(np.allclose(F.get_image(channel="right_image"), img_right)) def test_basic_channels_2(): img_left = np.zeros((100, 100, 3)) img_right = np.zeros((100, 100, 3))+255 F = labeled_movie.LabeledMovieFrame(internal_storage_method="jpg") F.create_channel("left_image") F.set_image(img_left, channel="left_image") F.create_channel("right_image") F.set_image(img_right, channel="right_image") F.set_default_channel("left_image") assert(np.allclose(F.get_image(), img_left)) assert(np.allclose(F.get_image(channel="left_image"), img_left)) assert(np.allclose(F.get_image(channel="right_image"), img_right)) if __name__ == "__main__": test_basic_channels_1() test_basic_channels_2() ================================================ FILE: PVM_tools/__init__.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== ================================================ FILE: PVM_tools/abstract_bounding_boxer.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ This module contains the abstract bounding boxer class. Import this module as: :: import PVM_tools.abstract_bounding_boxer or: :: from PVM_tools.abstract_bounding_boxer import AbstractBoundingBoxer """ from abc import ABCMeta, abstractmethod class AbstractBoundingBoxer(object): """ This is an abstract class defining all the methods a bounding boxer has to implement. """ __metaclass__ = ABCMeta @abstractmethod def set_current_bounding_box(self, bounding_region): """ Sets the current bounding region :param bounding_region: an object defining the region of interest :type bounding_region: PVM_tools.bounding_region.BoundingRegion .. note:: This is an abstract method, each bounding boxer needs to implement it """ @abstractmethod def process(self, heatmap, previous_bb=None): """ Processes new heatmap, tries to find the next bounding box :param heatmap: new probability/likelihood heatmap defining regions of interest :type heatmap: numpy.ndarray :param previous_bb: optional previous bounding box :type previous_bb: PVM_tools.bounding_region.BoundingRegion :return: """ @abstractmethod def reset(self): """ Reset the state of the bounding boxer :return: """ ================================================ FILE: PVM_tools/abstract_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ Abstract tracker ---------------- This module contains the basic definition of abstract or semi abstract types that any vision tracker should inherit. The main functionality of a tracker in encapsulated by two methods: 1. prime - used to inform the tracker which object or area its supposed to track 2. track - once primed, given an image, return the bounding box This module contains the abstract bounding boxer class. Import this module as: :: import PVM_tools.abstract_tracker or: :: from PVM_tools.abstract_tracker import AbstractVisionTracker, GenericVisionTracker """ from abc import ABCMeta, abstractmethod class AbstractVisionTracker(object): """ This is an abstract class defining all the methods a vision tracker has to implement. """ __metaclass__ = ABCMeta @abstractmethod def prime(self, im=None, bounding_box=None, **kwargs): """ :param im: Image as a numpy array or extension thereof :type im: numpy.ndarray :param bounding_box: instance of the bounding region class :type bounding_box: PVM_tools.bounding_region.BoundingRegion :param kwargs: additional implementation specific parameters Prime tracker on image and (optional) bounding box If im is None, priming will be done on the last tracked image kwargs can be passed on to particular implementations .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ @abstractmethod def track(self, im): """ :param im: numpy array representing an image :type im: numpy.ndarray :returns: bounding_box :rtype: PVM_tools.bounding_region.BoundingRegion Track on a given image .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ @abstractmethod def get_bbox(self): """ :returns: bounding_box :rtype: PVM_tools.bounding_region.BoundingRegion Return current bounding box of the target .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ @abstractmethod def get_heatmap(self, heatmap_name=None): """ :param heatmap_name: string object :type heatmap_name: string :returns: heatmap representing the probability of where the target object is :rtype: numpy.ndarray Return an internal heatmap for current tracker state. There may be several heatmaps available that can be requested specifying 'heatmap_name' parameter .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ class GenericVisionTracker(AbstractVisionTracker): """ Baseclass that describes default behaviors of most of the trackers. .. automethod:: _prime .. automethod:: _track """ @abstractmethod def _prime(self, im, bounding_box=None, **kwargs): """ The actual internal priming method called by prime. Trackers inheriting from GenericVisionTracker class need to implement this method. .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ def prime(self, im=None, bounding_box=None, **kwargs): """ :param im: Image as a numpy array or extension thereof :type im: numpy.ndarray :param bounding_box: bounding box (or in general any bounding region) of the target :type bounding_box: PVM_tools.bounding_region.BoundingRegion :param kwargs: additional implementation specific parameters Prime tracker on an image and (optional) an bounding box bounding box as an instance of PVM_tools.bounding_region.bounding_region class If im is None, priming will be done on the last tracked image kwargs can be passed on to particular implementations .. note:: Many trackers will require the bounding region object. Others may however be able to prime automatically e.g. base on the saliency of the object or prime on the object in the center of view etc. """ if im is None: if hasattr(self, '_last_tracked_im'): im = self._last_tracked_im else: raise Exception('Trying to prime on last tracked image, but no images were tracked yet') return self._prime(im, bounding_box, **kwargs) @abstractmethod def _track(self, im): """ The actual internal tracking method. Trackers inheriting from GenericVisionTracker need to implement this method. .. note:: This is an abstract method and needs to be implemented by inheriting classes. """ def track(self, im): """ Track the object of interest in the image. :param im: numpy array representing an image :type im: numpy.ndarray :returns: bounding_box :rtype: PVM_tools.bounding_region.BoundingRegion """ self.bbox = self._track(im) self._last_tracked_im = im return self.bbox def get_bbox(self): """ :returns: bounding_box :rtype: PVM_tools.bounding_region.BoundingRegion Return current bounding box of the target """ if hasattr(self, 'bbox'): return self.bbox else: return None def get_heatmap(self, heatmap_name=None): """ :param heatmap_name: string object :type heatmap_name: string :returns: heatmap representing the probability of where the target object is :rtype: numpy.ndarray Return an internal heatmap for current tracker state. There may be several heatmaps available that can be requested specifying 'heatmap_name' parameter This is default implementation - no heatmap available and None is returned. """ return None def get_name(self): """ Get the tracker name. :return: name of the tracker :rtype: str """ return self.name def finish(self): """ Call this method whenever the work with tracker has ended. This method will perform any necessary cleanup. .. note:: This call should be considered a part of destruction, that is after finish the object will not be used again and will be released. E.g. the tracker benchmark will create the tracker and reset if before each movie, but finish will be called at the end. Another example would be a tracker that is executing on a multi core machine using e.g. future encoder framework would use this call to stop all the worker processes and debug threads and so on. """ pass ================================================ FILE: PVM_tools/benchmark.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ This module contains the TrackerBenchmark class which encapsulates all the aspects of a tracker benchmark. Import this module as: :: import tracker_benchmark.benchmark or: :: from tracker_benchmark import TrackerBenchmark """ import matplotlib matplotlib.use('Agg') import PVM_tools.labeled_movie as lm import PVM_framework.PVM_Storage as PVM_Storage import argparse from PVM_tools.bounding_region import BoundingRegion import cv2 import traceback import sys import copy import numpy as np import matplotlib.pyplot as plt import cPickle import os import errno import time from matplotlib.backends.backend_pdf import PdfPages import PVM_framework.PVM_datasets as tracker_datasets class TrackerBenchmark(object): def __init__(self, output_dir="~/benchmark_results", resolution=None, have_display=True, draw_gt=True, perturb_gt=0, storage=None): """ :param output_dir: directory where the benchmarks will be stored on the local machine :param resolution: desired resolution of the benchmark :param have_display: should the display window be pulled :param draw_gt: should the ground truth bounding box be drawn on output videos :param perturb_gt: should there be a perturbed grond truth tracker added for baseline comparisons. If zero no tracker is added, if greater than zero it sets the fraction of perturbation as a fraction of ground truith box size. Values near 0.4 are reasonable. :return: The benchmark object """ self._raw_results = {} self._timing_results = {} self._processed_results = {} self._summary_results = {} self.draw_gt = draw_gt self.perturb_gt = perturb_gt self._colors = [(0, 0, 255), (0, 255, 255), (255, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 255), (0, 0, 128), (0, 128, 0), (128, 0, 0), (128, 255, 0), (0, 255, 128), (0, 128, 255), (255, 128, 0), (255, 0, 128), (128, 0, 255) ] self._styles = [{'c': 'r', 'l': '-'}, {'c': 'g', 'l': '-'}, {'c': 'b', 'l': '-'}, {'c': 'k', 'l': '-'}, {'c': 'c', 'l': '-'}, {'c': 'm', 'l': '-'}, {'c': 'y', 'l': '-'}, {'c': 'b', 'l': '-.'}, {'c': 'r', 'l': '-.'}, {'c': 'g', 'l': '-.'}, {'c': 'k', 'l': '-.'}, {'c': 'c', 'l': '-.'}, {'c': 'm', 'l': '-.'}, {'c': 'y', 'l': '-.'}, {'c': 'b', 'l': '--'}, {'c': 'r', 'l': '--'}, {'c': 'g', 'l': '--'}, {'c': 'k', 'l': '--'}, {'c': 'c', 'l': '--'}, {'c': 'm', 'l': '--'}, {'c': 'y', 'l': '--'}, {'c': 'b', 'l': ':'}, {'c': 'r', 'l': ':'}, {'c': 'g', 'l': ':'}, {'c': 'k', 'l': ':'}, {'c': 'c', 'l': ':'}, {'c': 'm', 'l': ':'}, {'c': 'y', 'l': ':'} ] self.resolution = resolution self._video = None self.have_display = have_display try: os.makedirs(os.path.expanduser(output_dir)) except OSError as err: if err.errno != errno.EEXIST: raise self._output_dir = os.path.expanduser(output_dir) if os.path.isfile(self._output_dir+"/raw_data.p"): if os.path.isfile(self._output_dir+"/timing_data.p"): self.load_results(filename=self._output_dir+"/raw_data.p", timing_filename=self._output_dir+"/timing_data.p") else: self.load_results(filename=self._output_dir+"/raw_data.p") self.storage = storage def evaluate_on_file(self, filepath, trackers, channel="default", targets=None, window=True, save_movie=True): """ Evaluates a given list of trackers on a given set of targets on a given channel. If target list is not given the trackers will be evaluated on all the targets available in the file. If a channel is not given, default channel will be used. :param filepath: path to the labeled movie file being evaluated :param trackers: list of tracker objects :param channel: channel to be evaluated ("default") :param window: show the window :param save_movie: save an avi file with the tracker performance. :return: new_results, False when all trackers have already been run on this file, otherwise True :rtype: bool """ for tracker in trackers: tracker.reset() fc = lm.FrameCollection() fc.load_from_file(filepath) name = os.path.basename(filepath) all_trackers = trackers if targets is None: targets = fc.Frame(0).get_targets() for target in targets: if name not in self._raw_results.keys(): self._raw_results[name] = {} self._timing_results[name] = {} if target not in self._raw_results[name].keys(): self._raw_results[name][target] = {} self._timing_results[name][target] = {} need_to_run_trackers = [] for tracker in trackers: if tracker.name not in self._raw_results[name][target]: need_to_run_trackers.append(tracker) if len(need_to_run_trackers) == 0: print "Skipping", name, target, "because it has already been evaluated for all trackers." return False # no new results if trackers != need_to_run_trackers: print "Warning, running only a subset of trackers can result in a different number of frames primed per run." trackers = need_to_run_trackers save_video_file = self._output_dir + "/" + name[:-4]+"_" + target + ".avi" if save_movie: self._video = cv2.VideoWriter() fps = 20 im = fc.Frame(0).get_image(channel=channel) if self.resolution is not None: im = cv2.resize(im, dsize=self.resolution) retval = self._video.open(os.path.expanduser(save_video_file), cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), fps, (im.shape[1], im.shape[0])) assert(retval) self._raw_results[name][target]["ground_truth"] = [] if self.perturb_gt>0: self._raw_results[name][target]['GT_pert_%2.2f' % self.perturb_gt] = [] for tracker in trackers: self._raw_results[name][target][tracker.name] = [] self._timing_results[name][target][tracker.name] = {} self._timing_results[name][target][tracker.name]['abs'] = [] self._timing_results[name][target][tracker.name]['norm'] = [] tracker.reset() primed = False for i in xrange(len(fc)): img = fc.Frame(i).get_image(channel=channel) if self.resolution is not None: old_shape = img.shape img = cv2.resize(img, dsize=self.resolution) pixels = np.prod(img.shape[0:2]) br = fc.Frame(i).get_label(channel=channel, target=target) if self.resolution is not None: br.scale_to_new_image_shape(new_image_shape=self.resolution, old_image_shape=old_shape) else: br.set_image_shape(img.shape) if not primed and br.empty: self._raw_results[name][target]['ground_truth'].append(BoundingRegion()) for tracker in trackers: self._raw_results[name][target][tracker.name].append(BoundingRegion()) if self.perturb_gt > 0: self._raw_results[name][target]['GT_pert_%2.2f' % self.perturb_gt].append(BoundingRegion()) continue if not primed: for tracker in trackers: any_except = False try: tracker.prime(img, br.copy()) except (KeyboardInterrupt, SystemExit): raise except: any_except = True exctype, value = sys.exc_info()[:2] if str(value).lower().find("prim") == -1: print "Caught Exception during priming" traceback.print_exc(file=sys.stdout) else: print "Caught Exception during priming", value if any_except: self._raw_results[name][target]['ground_truth'].append(br) for tracker in trackers: self._raw_results[name][target][tracker.name].append(br) continue else: primed = True disp_img = img.copy() self._raw_results[name][target]['ground_truth'].append(br) if self.perturb_gt>0: brp = br.copy() if not brp.empty: brp.scale(1.0+self.perturb_gt*0.5*(1.0-np.random.rand())) box = np.array(brp.get_box_pixels()) x_shift = int(box[2]*self.perturb_gt*0.5*(1.0-np.random.rand())) y_shift = int(box[3]*self.perturb_gt*0.5*(1.0-np.random.rand())) if self.resolution is not None: shape = self.resolution else: shape=img.shape box[0] = np.clip(box[0]+x_shift, 0, shape[0]) box[1] = np.clip(box[1]+y_shift, 0, shape[1]) box[2] = np.clip(box[2]+x_shift, 0, shape[0]) box[3] = np.clip(box[3]+y_shift, 0, shape[1]) pgt = BoundingRegion(image_shape=self.resolution, box=box) else: pgt = BoundingRegion() self._raw_results[name][target]['GT_pert_%2.2f' % self.perturb_gt].append(pgt) for (j, tracker) in enumerate(trackers): t_ = time.time() # Performance timing try: tbr = tracker.track(img) self._raw_results[name][target][tracker.name].append(tbr) except (KeyboardInterrupt, SystemExit): raise except: print "Exception in the tracker code! " + tracker.name, 'frame', i traceback.print_exc(file=sys.stdout) self._raw_results[name][target][tracker.name].append(BoundingRegion()) t = time.time() # Performance timing self._timing_results[name][target][tracker.name]['abs'].append(t-t_) self._timing_results[name][target][tracker.name]['norm'].append((t-t_)/pixels) if window or save_movie: for (j, tracker) in enumerate(all_trackers): tbr = self._raw_results[name][target][tracker.name][i] tbr.draw_box(disp_img, color=self._colors[(j+1) % len(self._colors)], thickness=1, annotation=tracker.name, linetype=cv2.CV_AA) if self.draw_gt: br.draw_box(disp_img, color=self._colors[0], thickness=1, annotation="Ground truth", linetype=cv2.CV_AA) if self.perturb_gt>0: pgt.draw_box(disp_img, color=self._colors[-1], thickness=1, annotation="Perturbed truth", linetype=cv2.CV_AA) if window and self.have_display: cv2.imshow("Tracking", disp_img) key = cv2.waitKey(1) & 0xFF if key == 27: break if save_movie: self._video.write(disp_img) if self.have_display and window: cv2.destroyAllWindows() if save_movie: self._video.release() return True # new results def save_results(self): """ :param filename: name of the file to be saved """ file = open(self._output_dir + "/raw_data.p", "wb") cPickle.dump(self._raw_results, file, protocol=-1) file.close() file = open(self._output_dir + "/timing_data.p", "wb") cPickle.dump(self._timing_results, file, protocol=-1) file.close() def load_results(self, filename, timing_filename=None): """ :param filename: path to the file loaded """ file = open(filename, "rb") self._raw_results = cPickle.load(file) file.close() if timing_filename is not None: file = open(timing_filename, "rb") self._timing_results = cPickle.load(file) file.close() self.process_results() def process_results(self, filename=None): """ Converts raw results into a more processed form :return: processed results, same indexing but other information """ print "Processing called with filename " + str(filename) sys.stdout.flush() if filename is None: filenames = self._raw_results.keys() else: filenames = [filename] for filename in filenames: if filename not in self._processed_results.keys(): self._processed_results[filename] = {} targets = self._raw_results[filename].keys() for target in targets: if target not in self._processed_results[filename].keys(): self._processed_results[filename][target] = {} tracker_names = copy.copy(self._raw_results[filename][target].keys()) num_of_frames = len(self._raw_results[filename][target]["ground_truth"]) for tracker in tracker_names: center_dists_pix = np.zeros(num_of_frames,)-0.00000001 center_dists_rel = np.zeros(num_of_frames,)-0.00000001 bounding_box_dists_rel = np.zeros((num_of_frames, 2))+np.Inf overlap = np.zeros(num_of_frames,) presence = np.zeros(num_of_frames,) target_present_and_tracked_frames = 0 target_absent_and_tracked_frames = 0 target_present_and_not_tracked_frames = 0 target_absent_and_not_tracked_frames = 0 for i in xrange(num_of_frames): if not self._raw_results[filename][target]["ground_truth"][i].empty and \ not self._raw_results[filename][target][tracker][i].empty: target_present_and_tracked_frames += 1 center_target = self._raw_results[filename][target]["ground_truth"][i].get_box_center_pixels() center_target_rel = self._raw_results[filename][target]["ground_truth"][i].get_box_center_relative() center_tracker = self._raw_results[filename][target][tracker][i].get_box_center_pixels() center_tracker_rel = self._raw_results[filename][target][tracker][i].get_box_center_relative() distance = np.sqrt((center_target[0]-center_tracker[0])**2 + (center_target[1]-center_tracker[1])**2) center_dists_pix[i] = distance distance_rel = np.sqrt((center_target_rel[0]-center_tracker_rel[0])**2 + (center_target_rel[1]-center_tracker_rel[1])**2) center_dists_rel[i] = distance_rel intersection = self._raw_results[filename][target]["ground_truth"][i].get_box_intersection(self._raw_results[filename][target][tracker][i]) overlap[i] = (intersection.get_area_pixels()*1.0 / (self._raw_results[filename][target]["ground_truth"][i].get_area_pixels() + self._raw_results[filename][target][tracker][i].get_area_pixels() - intersection.get_area_pixels() + 0.00001)) bb = self._raw_results[filename][target]["ground_truth"][i].get_box_pixels() shape_factor = self._raw_results[filename][target]["ground_truth"][i].image_shape_factor if shape_factor is None: # provide a reasonable default if shape_factor is not defined shape_factor = (1000, 1000) # bounding box width and height can be 0, which actually means a width of 1 pixel # shape_factor is the shape of the original image used to compute the bounding box # so we add 1.0/shape_factor to compensate and avoid division by 0 bounding_box_dists_rel[i, 0] = (center_target[0]-center_tracker[0])/float(bb[2]+1.0/shape_factor[0]) bounding_box_dists_rel[i, 1] = (center_target[1]-center_tracker[1])/float(bb[3]+1.0/shape_factor[1]) elif self._raw_results[filename][target]["ground_truth"][i].empty and \ not self._raw_results[filename][target][tracker][i].empty: target_absent_and_tracked_frames += 1 presence[i] = 1 elif not self._raw_results[filename][target]["ground_truth"][i].empty and \ self._raw_results[filename][target][tracker][i].empty: target_present_and_not_tracked_frames += 1 presence[i] = 2 else: target_absent_and_not_tracked_frames += 1 presence[i] = 3 bounding_box_dists_rel[i, :] = 0 true_positives = (target_present_and_tracked_frames * 100.0/num_of_frames) true_negatives = (target_absent_and_not_tracked_frames * 100.0/num_of_frames) false_positives = (target_absent_and_tracked_frames * 100.0/num_of_frames) false_negatives = (target_present_and_not_tracked_frames * 100.0/num_of_frames) self._processed_results[filename][target][tracker] = {} self._processed_results[filename][target][tracker]['total_frames'] = num_of_frames self._processed_results[filename][target][tracker]['total_true_positive'] = target_present_and_tracked_frames self._processed_results[filename][target][tracker]['total_true_negative'] = target_absent_and_not_tracked_frames self._processed_results[filename][target][tracker]['total_false_positive'] = target_absent_and_tracked_frames self._processed_results[filename][target][tracker]['total_false_negative'] = target_present_and_not_tracked_frames self._processed_results[filename][target][tracker]['true_positive'] = true_positives self._processed_results[filename][target][tracker]['true_negative'] = true_negatives self._processed_results[filename][target][tracker]['false_positive'] = false_positives self._processed_results[filename][target][tracker]['false_negative'] = false_negatives self._processed_results[filename][target][tracker]['presence_score'] = ((true_negatives+true_positives-false_negatives-false_positives + 100)/2) self._processed_results[filename][target][tracker]['distance_pix'] = center_dists_pix self._processed_results[filename][target][tracker]['distance_rel'] = center_dists_rel self._processed_results[filename][target][tracker]['bounding_box_dists_rel'] = bounding_box_dists_rel self._processed_results[filename][target][tracker]['overlap'] = overlap self._processed_results[filename][target][tracker]['presence'] = presence if target_present_and_tracked_frames>0: self._processed_results[filename][target][tracker]['avg_distance_pix'] = np.sum(center_dists_pix)/target_present_and_tracked_frames self._processed_results[filename][target][tracker]['avg_distance_rel'] = np.sum(center_dists_rel)/target_present_and_tracked_frames else: self._processed_results[filename][target][tracker]['avg_distance_pix'] = 1000000000.0 self._processed_results[filename][target][tracker]['avg_distance_rel'] = 1000000000.0 af = "all_files" at = "all_targets" if af not in self._processed_results.keys(): self._processed_results[af] = {} self._processed_results[af][at] = {} sys.stdout.flush() # The following bit is quite simple but ugly, needs to be abstracted probably. for filename in filenames: for target in self._processed_results[filename]: for tracker in self._processed_results[filename][target]: if tracker not in self._processed_results[af][at]: self._processed_results[af][at][tracker] = {} self._processed_results[af][at][tracker]['total_frames'] = 0 self._processed_results[af][at][tracker]['total_true_positive'] = 0 self._processed_results[af][at][tracker]['total_true_negative'] = 0 self._processed_results[af][at][tracker]['total_false_positive'] = 0 self._processed_results[af][at][tracker]['total_false_negative'] = 0 self._processed_results[af][at][tracker]['avg_distance_pix'] = 0 self._processed_results[af][at][tracker]['avg_distance_rel'] = 0 self._processed_results[af][at][tracker]['num_movies'] = 0 self._processed_results[af][at][tracker]['distance_pix'] = np.array([]) self._processed_results[af][at][tracker]['distance_rel'] = np.array([]) self._processed_results[af][at][tracker]['overlap'] = np.array([]) self._processed_results[af][at][tracker]['presence'] = np.array([]) self._processed_results[af][at][tracker]['bounding_box_dists_rel'] = np.zeros((0, 2)) N = self._processed_results[af][at][tracker]['num_movies'] self._processed_results[af][at][tracker]['total_frames'] += self._processed_results[filename][target][tracker]['total_frames'] self._processed_results[af][at][tracker]['total_true_positive'] += self._processed_results[filename][target][tracker]['total_true_positive'] self._processed_results[af][at][tracker]['total_true_negative'] += self._processed_results[filename][target][tracker]['total_true_negative'] self._processed_results[af][at][tracker]['total_false_positive'] += self._processed_results[filename][target][tracker]['total_false_positive'] self._processed_results[af][at][tracker]['total_false_negative'] += self._processed_results[filename][target][tracker]['total_false_negative'] self._processed_results[af][at][tracker]['true_positive'] = (self._processed_results[af][at][tracker]['total_true_positive'] * 100.0 / self._processed_results[af][at][tracker]['total_frames']) self._processed_results[af][at][tracker]['true_negative'] = (self._processed_results[af][at][tracker]['total_true_negative'] * 100.0 / self._processed_results[af][at][tracker]['total_frames']) self._processed_results[af][at][tracker]['false_positive'] = (self._processed_results[af][at][tracker]['total_false_positive'] * 100.0 / self._processed_results[af][at][tracker]['total_frames']) self._processed_results[af][at][tracker]['false_negative'] = (self._processed_results[af][at][tracker]['total_false_negative'] * 100.0 / self._processed_results[af][at][tracker]['total_frames']) self._processed_results[af][at][tracker]['presence_score'] = ((self._processed_results[af][at][tracker]['true_positive'] + self._processed_results[af][at][tracker]['true_negative'] - self._processed_results[af][at][tracker]['false_positive'] - self._processed_results[af][at][tracker]['false_negative'] + 100) / 2 ) self._processed_results[af][at][tracker]['avg_distance_pix'] = (self._processed_results[af][at][tracker]['avg_distance_pix'] * N+self._processed_results[filename][target][tracker]['avg_distance_pix'])/(N+1) self._processed_results[af][at][tracker]['avg_distance_rel'] = (self._processed_results[af][at][tracker]['avg_distance_rel'] * N+self._processed_results[filename][target][tracker]['avg_distance_rel'])/(N+1) self._processed_results[af][at][tracker]['num_movies'] = N+1 self._processed_results[af][at][tracker]['distance_pix'] = np.append(self._processed_results[af][at][tracker]['distance_pix'], self._processed_results[filename][target][tracker]['distance_pix']) self._processed_results[af][at][tracker]['distance_rel'] = np.append(self._processed_results[af][at][tracker]['distance_rel'], self._processed_results[filename][target][tracker]['distance_rel']) self._processed_results[af][at][tracker]['overlap'] = np.append(self._processed_results[af][at][tracker]['overlap'], self._processed_results[filename][target][tracker]['overlap']) self._processed_results[af][at][tracker]['presence'] = np.append(self._processed_results[af][at][tracker]['presence'], self._processed_results[filename][target][tracker]['presence']) self._processed_results[af][at][tracker]['bounding_box_dists_rel'] = \ np.concatenate((self._processed_results[af][at][tracker]['bounding_box_dists_rel'], self._processed_results[filename][target][tracker]['bounding_box_dists_rel']), axis=0) return self._processed_results def calc_precision(self, filename, target, tracker, thresholds, precision_level, absolute): data = self._processed_results[filename][target][tracker]['distance_pix'] data = data[np.where(data >= 0)] precision = np.zeros(thresholds.shape) for j, x in enumerate(thresholds): if absolute: precision[j] = 1.0*np.where(data < x)[0].shape[0]/self._processed_results[filename][target]["ground_truth"]['total_true_positive'] else: if self._processed_results[filename][target][tracker]['total_true_positive']>0: precision[j] = 1.0*np.where(data < x)[0].shape[0]/self._processed_results[filename][target][tracker]['total_true_positive'] else: precision[j] = 0 return precision, precision[np.argmin(np.abs(thresholds-precision_level))] def plot_precision(self, plot_individual_passes=False, precision_level=20, absolute=True): """ Plots the precision plots as described by [TRBEN13]_. .. [TRBEN13] Online Object Tracking: A Benchmark, Yi Wu and Jongwoo Lim and Ming-Hsuan Yang, IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2013 :return: """ if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] thresholds = np.arange(51) for filename in names: for target in self._processed_results[filename].keys(): fig = plt.figure(figsize=(7, 6)) ax = fig.add_subplot(111) trackers = sorted(self._processed_results[filename][target].keys()) precisions_at_level = np.zeros(len(trackers)) for i in range(len(trackers)): precision, precision_at_level = self.calc_precision(filename, target, trackers[i], thresholds, precision_level, absolute) precisions_at_level[i] = precision_at_level inds = np.argsort(precisions_at_level)[::-1] for i in inds: precision, precision_at_level = self.calc_precision(filename, target, trackers[i], thresholds, precision_level, absolute) plt.plot(thresholds, precision, label=trackers[i] + (" [%2.3f]" % precision_at_level), color=self._styles[i]['c'], linestyle=self._styles[i]['l']) if filename == "all_files": text = open(self._output_dir + ("/precision_20px_%s.txt" % trackers[i]), "w") text.write("%2.3f\n" % precision_at_level) text.close() plt.legend(loc=4, fontsize='x-small') plt.plot([20, 20], [0, 1], '--', color = '0.75') ax.set_title("Tracker performance precision plots %s, target %s" % (filename, target)) ax.set_xlabel("Distance threshold (in pixels)") ax.set_ylabel("Fraction of frames") if absolute: fig.savefig(self._output_dir + ("/precision_plot_%s_%s_abs.pdf" % (filename, target))) else: fig.savefig(self._output_dir + ("/precision_plot_%s_%s.pdf" % (filename, target))) def calc_precision_rel(self, filename, target, tracker, thresholds, precision_level, absolute): data = self._processed_results[filename][target][tracker]['distance_rel'] data = data[np.where(data >= 0)] precision = np.zeros(thresholds.shape) for j, x in enumerate(thresholds): if absolute: precision[j] = 1.0*np.where(data < x)[0].shape[0]/self._processed_results[filename][target]["ground_truth"]['total_true_positive'] else: if self._processed_results[filename][target][tracker]['total_true_positive']>0: precision[j] = 1.0*np.where(data < x)[0].shape[0]/self._processed_results[filename][target][tracker]['total_true_positive'] else: precision[j] = 0 return precision, precision[np.argmin(np.abs(thresholds-precision_level))] def plot_precision_rel(self, plot_individual_passes=False, precision_level=0.2, absolute=True): """ Plots a modified version of the precision plot, where the x-axis is in relative units instead of pixels :return: """ if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] thresholds = np.arange(0.0, 1.0, 0.001) for filename in names: for target in self._processed_results[filename].keys(): fig = plt.figure(figsize=(7, 6)) ax = fig.add_subplot(111) trackers = sorted(self._processed_results[filename][target].keys()) precisions_at_level = np.zeros(len(trackers)) for i in range(len(trackers)): precision, precision_at_level = self.calc_precision_rel(filename, target, trackers[i], thresholds, precision_level, absolute) precisions_at_level[i] = precision_at_level inds = np.argsort(precisions_at_level)[::-1] for i in inds: precision, precision_at_level = self.calc_precision_rel(filename, target, trackers[i], thresholds, precision_level, absolute) plt.plot(thresholds, precision, label=trackers[i] + (" [%2.3f]" % precision_at_level), color=self._styles[i]['c'], linestyle=self._styles[i]['l']) plt.legend(loc=4, fontsize='x-small') if filename == "all_files": text = open(self._output_dir + ("/precision_rel_%s.txt" % trackers[i]), "w") text.write("%2.3f\n" % precision_at_level) text.close() ax.set_title("Tracker performance precision plots %s, target %s" % (filename, target)) ax.set_xlabel("Distance threshold (relative)") ax.set_ylabel("Fraction of frames") if absolute: fig.savefig(self._output_dir + ("/precision_rel_plot_%s_%s_abs.pdf" % (filename, target))) else: fig.savefig(self._output_dir + ("/precision_rel_plot_%s_%s.pdf" % (filename, target))) def calc_success(self, filename, target, tracker, absolute): data = self._processed_results[filename][target][tracker]['overlap'] data = data[np.where(data >= 0)] steps = 100 success = np.zeros(steps,) xax = np.linspace(0, 1.0, steps) area = 0 for (i, x) in enumerate(xax): th = x if absolute: success[i] = 1.0*np.where(data > th)[0].shape[0]/self._processed_results[filename][target]["ground_truth"]['total_true_positive'] else: if self._processed_results[filename][target][tracker]['total_true_positive']>0: success[i] = 1.0*np.where(data > th)[0].shape[0]/self._processed_results[filename][target][tracker]['total_true_positive'] else: success[i] = 0 area += success[i]*1.0/steps return success, xax, area def plot_success(self, plot_individual_passes=False, absolute=True): """ Plots succes plots as described by [TRBEN13]_. :param plot_individual_passes: plot for individual movies, default True :return: """ if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] for filename in names: for target in self._processed_results[filename].keys(): fig = plt.figure(figsize=(7, 6)) ax = fig.add_subplot(111) trackers = sorted(self._processed_results[filename][target].keys()) areas = np.zeros(len(trackers)) for j in range(len(trackers)): success, xax, area = self.calc_success(filename, target, trackers[j], absolute) areas[j] = area # sort the trackers by their area score first, so that the legend will appear in rank order inds = np.argsort(areas)[::-1] for j in inds: success, xax, area = self.calc_success(filename, target, trackers[j], absolute) plt.plot(xax, success, label=trackers[j] + (" [%2.3f]" % area), color=self._styles[j]['c'], linestyle=self._styles[j]['l'] ) if filename == "all_files": text = open(self._output_dir + ("/success_%s.txt" % trackers[j]), "w") text.write("%2.3f\n" % area) text.close() plt.legend(fontsize='x-small') ax.set_title("Tracker performance success plots %s, target %s" % (filename, target)) ax.set_xlabel("Overlap threshold") ax.set_ylabel("Fraction of frames") if absolute: fig.savefig(self._output_dir + ("/success_plot_%s_%s_abs.pdf" % (filename, target))) else: fig.savefig(self._output_dir + ("/success_plot_%s_%s.pdf" % (filename, target))) def plot_performance(self, plot_individual_passes=True): """ Plots the tracker performance (overlap and relative distance to target) together with times of target absence and false positives and false negatives against time. :param plot_individual_passes: plot for individual movies, default True :return: """ names = self._raw_results.keys() if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] for filename in names: for target in self._processed_results[filename].keys(): pdf = PdfPages(self._output_dir + ("/performance_plot_%s_%s.pdf" % (filename, target))) for tracker in self._processed_results[filename][target].keys(): fig = plt.figure(figsize=(12, 10)) ax = fig.add_subplot(111) overlap = self._processed_results[filename][target][tracker]['overlap'] presence = self._processed_results[filename][target][tracker]['presence'] distance = self._processed_results[filename][target][tracker]['distance_rel'] xax = np.arange(0, overlap.shape[0], 1) plt.plot(xax, overlap, color='k', label="Overlap") plt.plot(xax, distance, color='k', linestyle=':', label="Distance") false_positive = np.zeros_like(presence) false_positive[np.where(presence == 1)] = np.sqrt(2) false_negative = np.zeros_like(presence) false_negative[np.where(presence == 2)] = np.sqrt(2) true_negative = np.zeros_like(presence) true_negative[np.where(presence == 3)] = np.sqrt(2) plt.plot(xax, false_positive, 'r', linewidth=0.1, zorder=1, label='False positive') plt.fill_between(xax, false_positive, 0, color='r', alpha=0.3) plt.plot(xax, false_negative, 'm', linewidth=0.1, zorder=1, label='False negative') plt.fill_between(xax, false_negative, 0, color='m', alpha=0.3) plt.plot(xax, true_negative, 'g', linewidth=0.1, zorder=1, label='True negative') plt.fill_between(xax, true_negative, 0, color='g', alpha=0.3) plt.legend(fontsize='x-small') ax.set_title("Tracker %s on %s, target %s" % (tracker, filename, target)) ax.set_xlabel("Time (frames)") ax.set_ylabel("Overlap/Distance") plt.close() pdf.savefig(fig) pdf.close() def plot_presence(self, plot_individual_passes=True): """ This method plots the presence pie chart, that is the pie chart containing the number of frames being true positives, true negatives, false positives and false negatives. Having good majority of true positives and true negatives against false positives and false negatives is a necessary but not sufficient condition of good tracking. :param plot_individual_passes: plot for individual movies, default True :return: """ names = self._raw_results.keys() if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] for filename in names: for target in self._processed_results[filename].keys(): pdf = PdfPages(self._output_dir + ("/presence_plot_%s_%s.pdf" % (filename, target))) for tracker in self._processed_results[filename][target].keys(): fig = plt.figure(figsize=(7, 6)) ax = fig.add_subplot(111) labels = ["True positive", "True negative", "False positive", "False negative"] sizes = [self._processed_results[filename][target][tracker]['true_positive'], self._processed_results[filename][target][tracker]['true_negative'], self._processed_results[filename][target][tracker]['false_positive'], self._processed_results[filename][target][tracker]['false_negative']] colors = ['yellowgreen', 'lightskyblue', 'red', 'magenta'] plt.pie(sizes, labels=labels, colors=colors, autopct="%1.1f%%", shadow=True) plt.axis('equal') ax.set_title("Tracker %s on %s, target %s" % (tracker, filename, target)) plt.close() pdf.savefig(fig) pdf.close() def plot_timing(self, plot_individual_passes=True): """ This method plots the timing plot for individual movie runs and combined over the entire dataset. :param plot_individual_passes: plot for individual movies, default True :return: """ names = self._timing_results.keys() combined_norm = {} for filename in names: for target in self._processed_results[filename].keys(): for tracker in self._processed_results[filename][target].keys(): if tracker == "ground_truth" or tracker.startswith("GT_pert"): continue timing = np.array(self._timing_results[filename][target][tracker]['norm']) if tracker not in combined_norm.keys(): combined_norm[tracker] = {"norm": timing} else: combined_norm[tracker]['norm'] = np.append(combined_norm[tracker]['norm'], timing) self._timing_results['all_files'] = {} self._timing_results['all_files']['all_targets'] = combined_norm names = self._timing_results.keys() for filename in names: for target in self._processed_results[filename].keys(): pdf = PdfPages(self._output_dir + ("/timing_plot_%s_%s.pdf" % (filename, target))) fig = plt.figure(figsize=(12, 10)) ax = fig.add_subplot(111) for (j, tracker) in enumerate(sorted(self._processed_results[filename][target].keys())): if tracker == "ground_truth" or tracker.startswith("GT_pert"): continue timing = np.array(self._timing_results[filename][target][tracker]['norm']) xax = np.arange(0, timing.shape[0], 1) plt.plot(xax, 1000*timing, label=tracker, color=self._styles[j]['c'], linestyle=self._styles[j]['l']) plt.legend(fontsize='x-small') plt.ylabel("Execution time (ms)") plt.xlabel("Time (frames)") plt.yscale('log') ax.set_title("Timing per pixel on %s, target %s" % (filename, target)) plt.close() pdf.savefig(fig) fig = plt.figure(figsize=(12, 10)) ax = fig.add_subplot(111) data = [] labels = [] for tracker in self._processed_results[filename][target].keys(): if tracker == "ground_truth" or tracker.startswith("GT_pert"): continue timing = np.array(self._timing_results[filename][target][tracker]['norm']) data.append(1000 * timing) labels.append(tracker) plt.boxplot(data, labels=labels, showmeans=True) plt.setp(ax.xaxis.get_majorticklabels(), rotation=15) plt.title("Tracker timing") try: plt.yscale('log') except: plt.yscale('linear') # some times log axis fails plt.ylabel("Execution time (ms)") pdf.savefig(fig) pdf.close() def calc_accuracy(self, filename, target, tracker, thresholds): data = np.abs(self._processed_results[filename][target][tracker]['bounding_box_dists_rel'])*2 # *2 so that 1.0 means edge of bounding box instead of 0.5 correct = np.zeros(thresholds.shape) for j, x in enumerate(thresholds): correct[j] = np.mean(np.bitwise_and(data[:, 0]<=x, data[:, 1]<=x)) return correct, correct[np.argmin(np.abs(thresholds-1.0))] # accuracy is measured at 1.0 def plot_accuracy(self, plot_individual_passes=False): """ Plots the accuracy of tracking as a function of distance away from center in units of bounding box size. If the center of the tracking bounding box lies within x*(ground truth bounding box) or the tracker correctly reports the absense of the target (regardless of x) the analysis reports the frame as correctly tracked, where x is varied between 0 and 10. When x==1, the ground truth bounding box is used. Note the area of the scaled bounding box grows as x^2. """ if plot_individual_passes: names = self._processed_results.keys() else: names = ["all_files"] thresholds = np.arange(0.0, 10.0, 0.01) for filename in names: for target in self._processed_results[filename].keys(): fig = plt.figure(figsize=(7, 6)) ax = fig.add_subplot(111) trackers = sorted(self._processed_results[filename][target].keys()) accuracies = np.zeros(len(trackers)) for i in range(len(trackers)): correct, accuracy = self.calc_accuracy(filename, target, trackers[i], thresholds) accuracies[i] = accuracy inds = np.argsort(accuracies)[::-1] for i in inds: correct, accuracy = self.calc_accuracy(filename, target, trackers[i], thresholds) plt.plot(thresholds, correct, label=trackers[i] + (" [%2.3f]" % accuracy), color=self._styles[i]['c'], linestyle=self._styles[i]['l']) if filename == "all_files": text = open(self._output_dir + ("/accuracy_%s.txt" % trackers[i]), "w") text.write("%2.3f\n" % accuracy) text.close() plt.legend(loc=4, fontsize='x-small') plt.plot([1, 1], [0, 1], '--', color = '0.75') ax.set_title("Tracker accuracy plots %s, target %s" % (filename, target)) ax.set_xlabel("bounding box linear size (1 means within ground truth bounding box)") ax.set_ylabel("Fraction of frames tracked correctly") fig.savefig(self._output_dir + ("/accuracy10_plot_%s_%s.pdf" % (filename, target))) ax.set_xlim(0, 2) fig.savefig(self._output_dir + ("/accuracy_plot_%s_%s.pdf" % (filename, target))) ax.set_xlim(0.03, 10) plt.xscale('log') fig.savefig(self._output_dir + ("/accuracy_log_plot_%s_%s.pdf" % (filename, target))) def print_results(self, file=sys.stdout, moviename=None): """ Prints the most basic processed results of tracker benchmark :param results: processed results as returned by process results :return: """ if moviename is None: flist = self._processed_results.keys() else: flist = [moviename] for moviename in flist: for target in self._processed_results[moviename].keys(): print >>file, "File " + moviename print >>file, "Target " + target for tracker in self._processed_results[moviename][target].keys(): print >>file, "Tracker " + tracker print >>file, "Total frames %d" % self._processed_results[moviename][target][tracker]['total_frames'] print >>file, "True positive %2.2f %%" % self._processed_results[moviename][target][tracker]['true_positive'] print >>file, "True negative %2.2f %%" % self._processed_results[moviename][target][tracker]['true_negative'] print >>file, "False positive %2.2f %%" % self._processed_results[moviename][target][tracker]['false_positive'] print >>file, "False negative %2.2f %%" % self._processed_results[moviename][target][tracker]['false_negative'] print >>file, "Presence score %2.2f %%" % self._processed_results[moviename][target][tracker]['presence_score'] print >>file, "Average distance %2.2f pixels" % self._processed_results[moviename][target][tracker]['avg_distance_pix'] print >>file, "Average distance rel %0.4f" % self._processed_results[moviename][target][tracker]['avg_distance_rel'] data = np.abs(self._processed_results[moviename][target][tracker]['bounding_box_dists_rel'])*2 # *2 so that 1.0 means edge of bounding box instead of 0.5 print >>file, "Correctly tracked %2.2f %%" % (100.0 * np.mean(np.bitwise_and(data[:, 0] <= 1.0, data[:, 1] <= 1.0))) def get_summary(self): self.print_results(moviename="all_files") def run(self, file_list, tracker_list, print_on_the_fly=True, channel="default", remove_downloads=True): """ Runs the evaluation on a list of files. Appends result dicts in a list :param file_list: list of files to be processed :param tracker_list: list of tracker objects to be evaluated :param print_on_the_fly: bool flag that will trigger instant performance message after each movie :return: list of result dicts """ summary = open(self._output_dir+"/summary.txt", "aw", 1) for (i, file) in enumerate(file_list): if file.endswith(".pkl"): print "Processing " + file + " %d of %d" % (i + 1, len(file_list)) local_path = self.storage.get(path=file) new_results = self.evaluate_on_file(local_path, tracker_list, channel=channel) if remove_downloads: try: os.remove(local_path) except OSError: print "Could not remove %s" % local_path if new_results: self.process_results(os.path.basename(local_path)) if print_on_the_fly: self.print_results(moviename=os.path.basename(local_path)) self.print_results(file=summary, moviename=os.path.basename(local_path)) self.save_results() # store the intermediate results for tracker in tracker_list: tracker.finish() self.print_results(file=summary, moviename="all_files") summary.close() def plot_all(self): self.plot_accuracy() self.plot_precision() self.plot_precision_rel() self.plot_success() self.plot_performance() self.plot_presence() self.plot_timing() def time_dir_name(prefix, suffix): lt = time.localtime() return prefix + ("_%d_%d_%d__%d_%d_" % (lt[0], lt[1], lt[2], lt[3], lt[4])) + suffix if __name__ == "__main__": have_display = True try: from Tkinter import Tk Tk() except: have_display = False print "No display detected, turning off visualizations." parser = argparse.ArgumentParser() parser.add_argument("-c", "--channel", type=str, default="default", help="Channel") parser.add_argument("-r", "--run", help="Run the benchmark (otherwise will just process data)", action="store_true") parser.add_argument("-o", "--output", type=str, default=time_dir_name("~/benchmark_results/run", ""), help="Results file to either store or load from") parser.add_argument("-disp", "--display", type=int, default=True, help="set to False to force no display") parser.add_argument("-m", "--sparse_manager_file", type=str, default="", help="SparseManager saved file to load") parser.add_argument("-s", "--set", help="Name of the dataset", type=str, default="green_ball") parser.add_argument("-res", "--resolution", help="resolution to force all videos to, eg. '96,96' or 'full', default 'full'", type=str, default="full") parser.add_argument("-rd", "--remove_downloads", help="removes downloads after use, default False", type=int, default=False) args = parser.parse_args() ts = PVM_Storage.Storage() res = None if args.resolution.lower() != "full": res = tuple(np.fromstring(args.resolution, dtype=int, sep=',')) if not args.display: have_display = False TB = TrackerBenchmark(output_dir=args.output, resolution=res, have_display=have_display, storage=ts) if args.run: files = [] for (file, channel) in tracker_datasets.sets["%s_testing" % args.set]: files.append(file) trackers = [] TB.run(files, trackers, channel=args.channel, remove_downloads=args.remove_downloads) TB.save_results() TB.get_summary() TB.plot_all() ================================================ FILE: PVM_tools/bounding_region.py ================================================ """ This module contains the class called BoundingRegion which encapsulates a number of methods used in any bounding box/region related computations. This class is extensively used in all of the trackers and tracker benchmark. Import this module as: :: import PVM_tools.bounding_region or: :: from PVM_tools.bounding_region import BoundingRegion """ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import cv2 import copy class BoundingRegion(object): """ A class used to pass return values from trackers as well as to initialize trackers :param image_shape: the shape of the image being used :param box: initializing box [left, top, width, height] either in pixel or relative coordinates :param contour: initializing contour in pixel or relative coordinates .. note:: Contour if given will override the box initialization, so the box will become the bounding box of the contour. If the contour is not given, it will be initialized as the set of points of the bounding box. :Example: :: import cv2 import numpy as np shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) d = bounding_region(shape, contour=np.array([[[10, 20]], [[25, 15]], [[80, 65]], [[60, 70]], [[20, 75]], [[5, 50]]])) d.draw_contour(image, color=(0, 255, 0)) d.draw_box(image) min_rect = d.get_min_area_rect_pixels() cv2.drawContours(image, [np.int0(cv2.cv.BoxPoints(min_rect))], 0, (0, 255, 255)) circle = d.get_min_enclosing_circle_pixels() cv2.circle(image, circle[0], circle[1], (255, 0, 0)) cv2.ellipse(image, d.get_ellipse_fit_pixels(), (255, 255, 255)) cv2.imshow("image", image) cv2.waitKey(0) A BoundingRegion object can also be initilized with a box (rectangle): :Example: :: shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) b = BoundingRegion(shape, box=np.array([10, 20, 15, 45])) b.draw_box(image) cv2.ellipse(image, b.get_ellipse_fit_pixels(), (255, 255, 255)) cv2.imshow("image", image) cv2.waitKey(0) When a BoundingRegion object is initiated without parameters, it will assume it is empty. The property BoundingRegion.empty will be set to true, and object methods will either be returning None or have no effect. :Example: :: e = BoundingRegion() box = e.get_box_pixels() # box is None e.draw_box(image) # nothing happens to the image """ def __init__(self, image_shape=None, box=None, contour=None, confidence=0.0): self._version = 0.1 self._confidence = confidence self._is_keyframe = False self.image_shape = image_shape self.box_is_primary = False self.box = None self.contour = None self._empty = False if image_shape is not None: self.set_image_shape(image_shape) if box is None and contour is None: self._empty = True return if box is not None: if type(box) != np.ndarray: # if a list of floats are passed in this will fail box = np.array(box, dtype=np.int32) if type(box) == BoundingRegion: box = box.get_box_pixels() if np.any(np.isinf(box)) or np.any(np.isnan(box)): raise Exception("Bounding Box contains inf or nan") if np.min(box[2:]) < 0: self._empty = True return if np.issubdtype(box.dtype, np.float): # box is stored in pixel coordinates, for now at least assert np.max(np.abs(box)) < 1.5, "relative bounding box values used but are much larger than 1.0" self.box = (box * self.image_shape_factor).astype(np.int32) else: self.box = box self.box_is_primary = True if contour is not None: if type(contour) == BoundingRegion: contour = contour.get_contour_pixels() if np.issubdtype(contour.dtype, np.float): contour_int = np.zeros_like(contour, dtype=np.int32) for i in contour.shape[0]: contour_int[i][0][0] = int(self.image_shape[1]*contour[i][0][0]) contour_int[i][0][1] = int(self.image_shape[0]*contour[i][0][1]) self.contour = contour_int else: self.contour = contour self._box_from_contour() else: self._contour_from_box() self._update_internals() def _contour_from_box(self): contour = np.zeros((4, 1, 2), dtype=np.int32) contour[0][0][0] = self.box[0] contour[0][0][1] = self.box[1] contour[1][0][0] = self.box[0]+self.box[2] contour[1][0][1] = self.box[1] contour[2][0][0] = self.box[0]+self.box[2] contour[2][0][1] = self.box[1]+self.box[3] contour[3][0][0] = self.box[0] contour[3][0][1] = self.box[1]+self.box[3] self.contour = contour self.box_is_primary = True def _box_from_contour(self): self.box_is_primary = False self.box = np.array(cv2.boundingRect(self.contour)) - np.array([0, 0, 1, 1]) def _update_internals(self): if not self._empty: self.moments = cv2.moments(self.contour) def set_image_shape(self, shape): """ Sets the image shape. Image shape is nescessary to calculate the relative coordinates. :param shape: tuple, e.g. (100, 100, 3) :return: """ self.image_shape = shape self.image_shape_factor = np.array(list(self.image_shape[1::-1])*2) def get_box_pixels(self): """ Returns the bounding box in image pixel coordinates or None if the object is empty :return: bounding box [t, l, w, h] :rtype: numpy.ndarray of int32 """ if self._empty: return None return (self.box[0], self.box[1], self.box[2], self.box[3]) def get_box_relative(self): """ Returns the bounding box in relative coordinates or None if the object is empty :return: bounding box [t, l, w, h] :rtype: numpy.ndarray of float """ if self._empty: return None if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") return self.box.astype(np.float) / self.image_shape_factor def get_contour_pixels(self): """ Returns the bounding contour in image pixel coordinates or None if the object is empty :return: contour, array N x 1 x 2 representing the contour as a collection of N points :rtype: numpy.ndarray of int32 """ if self._empty: return None return self.contour def get_contour_relative(self): """ Returns the contour in relative coordinates or None if the object is empty :return: contour, array N x 1 x 2 representing the contour as a collection of N points :rtype: numpy.ndarray of float """ if self._empty: return None if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") contour_f = np.zeros_like(self.contour, dtype=np.float) for i in xrange(self.contour.shape[0]): contour_f[i][0][0] = self.contour[i][0][0]*1.0/self.image_shape[1] contour_f[i][0][1] = self.contour[i][0][1]*1.0/self.image_shape[0] return contour_f def get_box_center_pixels(self): """ Returns the center of the box in pixel coordinates or None if the object is empty :return: center tuple (x, y) of int :rtype: 2-tuple of integers """ if not self._empty: return self.box[0] + self.box[2] / 2, self.box[1] + self.box[3] / 2 else: return None def get_box_center_relative(self): """ Returns the center of the box in pixel coordinates or None if the object is empty :return: center tuple (x, y) of float :rtype: 2-tuple of floats """ if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") if not self._empty: (bx, by) = self.get_box_center_pixels() return (bx *1.0 / self.image_shape[1], by * 1.0 / self.image_shape[0]) else: return None def get_centroid_pixels(self): """ Returns the centroid of the contour in pixel coordinates or None if the object is empty :return: centroid tuple (x, y) of int :rtype: 2-tuple of integers """ if self._empty: return None cx = int(self.moments['m10']/self.moments['m00']) cy = int(self.moments['m01']/self.moments['m00']) return cx, cy def get_centroid_relative(self): """ Returns the centroid of the contour in relative coordinates. Returns None if the object is empty. :return: centroid tuple (x, y) of float :rtype: 2-tuple of floats """ if self._empty: return None if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") (cx, cy) = self.get_centroid_pixels() return (cx * 1.0 / self.image_shape[1], cy * 1.0 / self.image_shape[0]) def scale(self, factor_x, factor_y=None): """ Expand or contract the bounding box or contour around its center by a given factor :param factor_x: The multiplicative scale parameter in the x direction :type factor_x: float :param factor_y: The multiplicative scale parameter in the y direction :type factor_y: float .. note:: if factor_y parameter is omitted, then the factor_x is used in both directions .. note:: The scaling is done with respect to the contour's centroid as computed by the get_centroid methods. :Example: :: shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) d = bounding_region(shape, contour=np.array([[[10, 20]], [[25, 15]], [[80, 65]], [[60, 70]], [[20, 75]], [[5, 50]]])) d.draw_contour(image, color=(0, 255, 0)) # Scale to half the size d.scale(0.5) d.draw_contour(image, color=(255, 255, 0)) d.draw_box(image) cv2.imshow("Two contours", image) cv2.waitKey(0) """ if self._empty: return if factor_y is None: factor_y = factor_x if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") if self.box_is_primary: shift_x = self.box[2] * (1.-factor_x) * 0.5 shift_y = self.box[3] * (1.-factor_y) * 0.5 self.box = np.array([np.maximum(self.box[0]+shift_x, 0), np.maximum(self.box[1]+shift_y, 0), np.minimum(self.box[2]*factor_x, self.image_shape[1]), np.minimum(self.box[3]*factor_y, self.image_shape[0])]).astype(np.int32) self._contour_from_box() self._update_internals() else: (cx, cy) = self.get_centroid_pixels() new_contour = np.zeros_like(self.contour, dtype=np.int32) for i in xrange(self.contour.shape[0]): new_contour[i][0][0] = np.clip(int(cx + (self.contour[i][0][0]-cx)*factor_x), a_min=0, a_max=self.image_shape[1]) new_contour[i][0][1] = np.clip(int(cy + (self.contour[i][0][1]-cy)*factor_y), a_min=0, a_max=self.image_shape[0]) self.contour = new_contour self._box_from_contour() self._update_internals() def draw_box(self, image, color=(0, 0, 255), thickness=2, annotation=None, linetype=8): """ Draws the bounding box on an image :param image: image to place the drawing (RGB) :type image: numpy.ndarray (w, h, 3) :param color: 3-tuple representing RGB of the color :type color: 3-tuple :param thickness: line thickness (default=2) :type thickness: int :param annotation: optional annotation (e.g. name of the tracker) :type annotation: basestring :return: No return value :Example: :: shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) d = bounding_region(shape, contour=np.array([[[10, 20]], [[25, 15]], [[5, 50]]])) d.draw_box(image, color=(0, 255, 0), annotation="This is a box") """ if self._empty: return cv2.rectangle(image, (self.box[0], self.box[1]), (self.box[0] + self.box[2], self.box[1] + self.box[3]), color=color, thickness=thickness, lineType=linetype) if annotation is not None: cv2.putText(image, text=annotation, org=(self.box[0], self.box[1]+8), fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=0.7, color=color, lineType=linetype) def draw_contour(self, image, color=(0, 0, 255), thickness=2, linetype=8): """ Draws the bounding contour on an image :param image: image to place the drawing (RGB) :type image: numpy.ndarray (w, h, 3) :param color: 3-tuple representing RGB of the color :type color: 3-tuple :param thickness: line thickness (default=2) :type thickness: int :return: No return value """ if self._empty: return cv2.drawContours(image, [self.contour], 0, color=color, thickness=thickness, lineType=linetype) def get_min_area_rect_pixels(self): """ Calculates the minimum enclosing area rectangle (can be rotated). Returns None if the object is empty. :return: tuple composed of (x, y) of the center, (w, h) and the angle ((x, y), (w, h), angle) :rtype: 3-tuple """ if self._empty: return None rect = cv2.minAreaRect(self.contour) return rect def get_min_enclosing_circle_pixels(self): """ Calculates the minimum enclosing circle. Returns None if the object is empty. :return: tuple composed of (x, y) of the center and the radius ((x, y), radius) :rtype: 3-tuple of integer """ if self._empty: return None circle = cv2.minEnclosingCircle(self.contour) return (int(circle[0][0]), int(circle[0][1])), int(circle[1]) def get_ellipse_fit_pixels(self): """ Calculates the minimum enclosing ellipse """ if self._empty: return None ellipse = cv2.fitEllipse(self.contour) return ellipse def get_area_pixels(self): """ Returns the area of the enclosing contour in pixels^2 or None if the object is empty :return: area :rtype: int """ if self._empty: return 0 area = cv2.contourArea(self.contour) return area def get_area_relative(self): """ Returns the area of the enclosing contour in fraction of the area of the whole image or None if the object is empty :return: area :rtype: float """ if self._empty: return None if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") area = self.get_area_pixels() return float(area)/(self.image_shape[0]*self.image_shape[1]) def get_perimeter_pixels(self): """ Returns the perimeter of the enclosing contour in pixels or None if the object is empty :return: perimeter :rtype: int """ if self._empty: return None perimeter = cv2.arcLength(self.contour, closed=True) return perimeter @property def empty(self): """ Returns true if the bounding box is empty. If the object is empty many methods will return None or have no effect. :return: True | False :rtype: Boolean """ return self._empty @property def confidence(self): """ Returns the optional confidence that the region is indeed right, if the tracking algorithm did provide it. :return: confidence :rtype: float """ return self._confidence def get_mask(self, contour=False): """ Returns a mask, single channel uint8 numpy array with values 255 inside the bounding box or bounding contour and zeros elsewhere. :param contour: if true the bounding contour will be painted, otherwise just the bounding box. Default false. :type contour: bool :return: mask :rtype: numpy.ndarray """ if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") mask = np.zeros((self.image_shape[0], self.image_shape[1]), dtype=np.uint8) if self._empty: return mask if contour: cv2.drawContours(mask, [self.contour], 0, color=255, thickness=-1) else: cv2.rectangle(mask, (self.box[0], self.box[1]), (self.box[0] + self.box[2], self.box[1] + self.box[3]), color=255, thickness=-1) return mask def get_box_intersection(self, bounding_region): """ Given a bounding_region object computes and returns a new BoundingRegion that corresponds to the intersection of the bounding box of the current object with the box of the region given as argument. Retuns an empty BoundingRegion if the interseciton is empty. :param bounding_region: A BoundingRegion object to compute intersection with :type bounding_region: BoundingRegion :return: Bounding region of the intersection of the boxes :rtype: BoundingRegion """ x1_1 = self.box[0] y1_1 = self.box[1] x1_2 = self.box[0] + self.box[2] y1_2 = self.box[1] + self.box[3] box2 = bounding_region.get_box_pixels() x2_1 = box2[0] y2_1 = box2[1] x2_2 = box2[0] + box2[2] y2_2 = box2[1] + box2[3] x3_1 = max(x1_1, x2_1) y3_1 = max(y1_1, y2_1) width = max(-1, min(x1_2, x2_2) - x3_1) height = max(-1, min(y1_2, y2_2) - y3_1) if width * height >= 0: return BoundingRegion(image_shape=self.image_shape, box=(x3_1, y3_1, width, height)) else: return BoundingRegion() def copy(self): """ Returns a copy of the object. :return: copy :rtype: BoundingRegion """ return copy.deepcopy(self) def scale_to_new_image_shape(self, new_image_shape, old_image_shape=None): """ Converts a bounding region to an instance which corresponds to the same relative size of the bounding box in a new size image. E.g. a bounding region may correspond to a box (100, 100, 50, 50) in an 1080x720 frame. Now the frame is scaled to 540x360, the new bounding region should be scaled to (50, 50, 25, 25) to correspond with the same area in the image. :param new_image_shape: the new shape of the image :type new_image_shape: tuple :param old_image_shape: old shape (optional if not given earlier) :type old_image_shape: tuple """ if self.image_shape is None and old_image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") if old_image_shape is None: old_image_shape = self.image_shape if self._empty: return factor_x = new_image_shape[1]*1.0/old_image_shape[1] factor_y = new_image_shape[0]*1.0/old_image_shape[0] self.image_shape = new_image_shape self.image_shape_factor = np.array(list(self.image_shape[1::-1])*2) if self.box_is_primary: self.box = np.array([np.maximum(self.box[0]*factor_x, 0), np.maximum(self.box[1]*factor_y, 0), np.minimum(self.box[2]*factor_x, new_image_shape[1]), np.minimum(self.box[3]*factor_y, new_image_shape[0])]).astype(np.int32) self._contour_from_box() self._update_internals() else: new_contour = np.zeros_like(self.contour, dtype=np.int32) for i in xrange(self.contour.shape[0]): new_contour[i][0][0] = np.clip(int((self.contour[i][0][0])*factor_x), a_min=0, a_max=new_image_shape[1]) new_contour[i][0][1] = np.clip(int((self.contour[i][0][1])*factor_y), a_min=0, a_max=new_image_shape[0]) self.contour = new_contour self._box_from_contour() self._update_internals() def get_version(self): """ Gets the version of the bounding region :return: """ if hasattr(self, "_version"): return self._version else: return 0 def is_keyframe(self): """ Is a keyframe :return: """ if hasattr(self, "_is_keyframe"): return self._is_keyframe else: return False def set_keyframe(self, key_status): """ Set keyframe status :param key_status: :return: """ self._is_keyframe = key_status if __name__ == "__main__": shape = (100, 100, 3) image = np.zeros(shape) b = BoundingRegion(shape, box=np.array([10, 20, 15, 45])) f = BoundingRegion(shape, box=np.array([10, 20, 15, 45])) c = BoundingRegion(shape, contour=np.array([[[10, 20]], [[25, 20]], [[25, 65]], [[10, 65]]])) assert(np.allclose(b.get_box_pixels(), c.get_box_pixels())) assert(np.allclose(b.get_contour_pixels(), c.get_contour_pixels())) assert(np.allclose(b.get_contour_relative(), c.get_contour_relative())) assert(b.get_box_center_pixels() == c.get_box_center_pixels()) assert(b.get_box_center_relative() == c.get_box_center_relative()) assert(b.get_centroid_pixels() == c.get_centroid_pixels()) assert(b.get_centroid_relative() == c.get_centroid_relative()) assert(b.get_area_pixels() == c.get_area_pixels()) assert(b.get_area_relative() == c.get_area_relative()) assert(b.get_perimeter_pixels() == c.get_perimeter_pixels()) print b.get_box_pixels() print b.get_contour_pixels() print b.get_contour_relative() print b.get_box_center_pixels() print b.get_box_center_relative() print b.get_centroid_pixels() print b.get_centroid_relative() print b.get_area_pixels() print b.get_area_relative() print b.get_perimeter_pixels() f.scale(0.5) b.draw_box(image) f.draw_box(image, (255, 255, 0)) f.scale(2) f.draw_box(image, (255, 255, 0)) cv2.imshow("image", image) cv2.waitKey(0) image = np.zeros(shape) shape = (100, 100, 3) d = BoundingRegion(shape, contour=np.array([[[10, 20]], [[25, 15]], [[80, 65]], [[60, 70]], [[20, 75]], [[5, 50]]])) d.draw_contour(image, color=(0, 255, 0), thickness=-1) d.scale(0.5) d.draw_contour(image, color=(255, 255, 0)) d.draw_box(image) cv2.drawContours(image, [np.int0(cv2.cv.BoxPoints(d.get_min_area_rect_pixels()))], 0, (0, 255, 255)) circle = d.get_min_enclosing_circle_pixels() cv2.circle(image, circle[0], circle[1], (255, 0, 0)) cv2.ellipse(image, d.get_ellipse_fit_pixels(), (255, 255, 255)) cv2.imshow("image1", image) cv2.waitKey(0) ================================================ FILE: PVM_tools/labeled_movie.py ================================================ """ This module contains a number of classes representing handling, loading and encoding a movie object with labels. The classes included are: 1. LabeledMovieFrame - the primary container for images, labels and other data 2. LabeledMovieHeader - header object used in writing files 3. LabeledMovieReader - Reader object for dealing with files 4. LabeledMovieWriter - Writer object for dealing with files 5. FrameCollecion - a container object representing a movie when it is loaded in memory In general in most of the cases it will be sufficient to load the FrameCollecion object into any project dealing with labeled movies, as it has methods for loading, saving and accessing frames. Import this module as: :: import PVM_tools.labeled_movie or: :: from PVM_tools.labeled_movie import LabeledMovieFrame, LabeledMovieHeader, \\ LabeledMovieReader, LabeledMovieWriter, FrameCollection """ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cPickle import PVM_framework.CoreUtils as CoreUtils import cv2 import numpy as np import time class LabeledMovieFrame(object): """ :param internal_storage_method: Storage method 'raw'|'png'|'jpg' :type internal_storage_method: str :param compression_level: A number 0-100 describing the level of compression. Lower is more compressed. :type compression_level: int A generic class representing a movie frame. As frame is an object composed of multiple elements: * One or more channels * One or more images (each assigned to a channel) * One or more bounding regions (labels) each assigned to a channel * One or more time stamps, each assigned to a channel * One or more metadata strings, each ssigned to a channel There always exist a default channel in a frame. That default channel can be redirected to any other channel created by the user. E.g. there may exist a frame which has two images (e.g left and right taken from a stereo camera). The default channel will point to one of them e.g. left. The internal storage algorithm for images can be selected from: * 'raw' - no compression, raw representation * 'png' - png compression algorithm * 'jpg' - JPEG compression algorithm (loses information) jpg is recommended for long and/or large movies, for practical reasons. It can save ~10x the space, but the image will lose some quality. :Example: :: cap = cv2.VideoCapture(-1) ret, img = cap.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.set_image(img) """ def __init__(self, internal_storage_method="raw", compression_level=80): self._version = 0.4 if internal_storage_method not in ["raw", "png", "jpg"]: raise "Unsupported storage method" self._internal_storage_method = internal_storage_method self._per_channel_storage_method = {} self._image_storage = {} self._audio_storage = {} self._label_storage = {} self._timestamp_storage = {} self._metadata_storage = {} self._compression_level = np.clip(compression_level, 0, 99) self._channel = {"default": "default"} def create_channel(self, channel): """ :param channel: Name of the channel to create :type channel: str Creates a new channel. A channel is a container which can take an image and other attributes. :Example: :: cap0 = cv2.VideoCapture(0) cap1 = cv2.VideoCapture(1) ret, img_left = cap0.read() ret, img_right = cap1.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.create_channel("left_image") F.set_image(img_left, channel="left_image") F.create_channel("right_image") F.set_image(img_right, channel="right_image") F.set_default_channel("left_image") assert(np.allclose(F.get_image(), img_left)) """ self._channel[channel] = channel self._per_channel_storage_method[channel] = self._internal_storage_method def set_default_channel(self, channel, rename_previous_default="channel01"): """ :param channel: Name of the channel to create :type channel: str :param rename_previous_default: Optional new name of the previous default channel :type channel: str Set the default channel. A channel is a container which can take an image and other attributes. A default channel is the one that will be used if the channel parameter is omitted. In certain cases only the default channel existed in the frame, new channel got created and is set to default. In this case the previous content of the default channel could become inaccessible. In such case the previous default channel will be renamed with 'rename_previous_default' name (default 'channel01'). :Example: :: cap0 = cv2.VideoCapture(0) cap1 = cv2.VideoCapture(1) ret, img_left = cap0.read() ret, img_right = cap1.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.create_channel("left_image") F.set_default_channel("left_image") F.set_image(img_left) # Now left_image is the default channel F.create_channel("right_image") F.set_image(img_right, channel="right_image") assert(np.allclose(F.get_image(), img_left)) """ if "default" in self._channel.keys(): self._channel[rename_previous_default] = self._channel["default"] self._channel["default"] = channel def set_image(self, image, channel="default", storage_method=None): """ :param image: numpy array containing an RGB representation of an image :type image: numpy.ndarray :param channel: optional argument defining with which channel the image should be associated (default: "default") :param channel: str :return: Integer indicating whether the processing of the image was successful :rtype: int Inserts an image into a channel. """ ret = 0 if storage_method is None: self._per_channel_storage_method[channel] = self._internal_storage_method elif storage_method in ["raw", "png", "jpg"]: self._per_channel_storage_method[channel] = storage_method else: raise Exception("Unknown storage method") if self._per_channel_storage_method[channel] == "raw": self._image_storage[self._channel[channel]] = image.copy() ret = 0 if self._per_channel_storage_method[channel] == "png": quality = 9 - int(self._compression_level/10) (ret, buf) = cv2.imencode(".png", image, (cv2.IMWRITE_PNG_COMPRESSION, quality)) self._image_storage[self._channel[channel]] = buf if self._per_channel_storage_method[channel] == "jpg": (ret, buf) = cv2.imencode(".jpg", image, (cv2.IMWRITE_JPEG_QUALITY, self._compression_level)) self._image_storage[self._channel[channel]] = buf return ret def set_label(self, label, channel="default", target="default"): """ :param label: A serializable object containing information about the frame. E.g. PVM_tools.BoundingRegion :type label: object :param channel: optional argument defining with which channel the image should be associated (default: "default") :type channel: str :param target: name of the target (there may be many labels/targets on a single image) :type target: str Inserts an label into a channel. """ if self._channel[channel] not in self._label_storage.keys(): self._label_storage[self._channel[channel]] = {} self._label_storage[self._channel[channel]][target] = label def set_timestamp(self, timestamp, channel="default"): """ :param timestamp: A serializable object containing time information. E.g. time as float. :type timestamp: object :param channel: optional argument defining with which channel the image should be associated (default: "default") :param channel: str Inserts an timestamp into a channel. """ self._timestamp_storage[self._channel[channel]] = timestamp def set_metadata(self, metadata, channel="default"): """ :param metadata: A serializable object containing metadata information. E.g. a string. :type metadata: object :param channel: optional argument defining with which channel the image should be associated (default: "default") :param channel: str Inserts a metadata into a channel. """ self._metadata_storage[self._channel[channel]] = metadata def set_audio(self, audio, channel="default"): """ :param audio: A serializable object containing audio information. :type audio: object :param channel: optional argument defining with which channel the image should be associated (default: "default") :param channel: str Inserts an audio information into a channel. """ self._audio_storage[self._channel[channel]] = audio def get_channels(self): """ Retuns the names of channels available :return: list of channels :rtype: list """ return self._channel.keys() def get_image(self, channel="default"): """ :param channel: optional argument selecting the channel (will use default channel if omitted) :type channel: str :return: decoded image in RGB format :rtype: numpy.ndarray Decodes and returns the image from the given channel """ if self._per_channel_storage_method[self._channel[channel]] == "raw": return self._image_storage[self._channel[channel]] if self._per_channel_storage_method[self._channel[channel]] == "png": im_data = self._image_storage[self._channel[channel]] return cv2.imdecode(im_data, cv2.CV_LOAD_IMAGE_COLOR) if self._per_channel_storage_method[self._channel[channel]] == "jpg": im_data = self._image_storage[self._channel[channel]] return cv2.imdecode(im_data, cv2.CV_LOAD_IMAGE_COLOR) def get_label(self, channel="default", target="default"): """ :param channel: optional argument selecting the channel (will use default channel if omitted) :type channel: str :return: label object :rtype: object Decodes and returns the label object from the given channel. Returns None if no label is present. """ try: return self._label_storage[self._channel[channel]][target] except KeyError: return None def get_targets(self, channel="default"): """ :param channel: identifier of the channel :type channel: str :return: list of target labels :type: list Returns the list of target names. """ try: return self._label_storage[self._channel[channel]].keys() except: return None def get_audio(self, channel="default"): """ :param channel: optional argument selecting the channel (will use default channel if omitted) :type channel: str :return: audio object :rtype: object Decodes and returns the audio object from the given channel """ try: return self._audio_storage[self._channel[channel]] except KeyError: return None def get_timestamp(self, channel="default"): """ :param channel: optional argument selecting the channel (will use default channel if omitted) :type channel: str :return: timestamp object :rtype: object Decodes and returns the timestamp object from the given channel """ try: return self._label_storage[self._channel[channel]] except KeyError: return None def get_metadata(self, channel="default"): """ :param channel: optional argument selecting the channel (will use default channel if omitted) :type channel: str :return: metadata object :rtype: object Decodes and returns the metadata object from the given channel """ try: return self._metadata_storage[self._channel[channel]] except KeyError: return None @property def version(self): return self._version @property def storage_method(self): if self._version < 0.3: return self.internal_storage_method else: return self._internal_storage_method @property def compression(self): if self._version < 0.3: return self.compression_level else: return self._compression_level @classmethod def upgrade_to_latest_version(cls, Frame): if Frame.version < 0.3: storage_method = Frame.internal_storage_method compression = Frame.compression_level New_frame = LabeledMovieFrame(internal_storage_method=storage_method, compression_level=compression) channels = Frame.channel.keys() for channel in channels: New_frame.set_image(image=Frame.get_image(channel=channel), channel=channel) try: label = Frame._label_storage[channel] except KeyError: label = None if label is not None: New_frame.set_label(label, channel=channel) audio = Frame.get_audio(channel=channel) if audio is not None: New_frame.set_audio(audio, channel=channel) return New_frame elif Frame.version < 0.4: storage_method = Frame.storage_method compression = Frame.compression New_frame = LabeledMovieFrame(internal_storage_method=storage_method, compression_level=compression) channels = Frame.get_channels() for channel in channels: New_frame.set_image(image=Frame.get_image(channel=channel), channel=channel) label = Frame.get_label(channel=channel) if label is not None: New_frame.set_label(label, channel=channel) audio = Frame.get_audio(channel=channel) if audio is not None: New_frame.set_audio(audio, channel=channel) return New_frame else: return Frame def __setstate__(self, state): self.__dict__ = state if self._version < 0.3: self._internal_storage_method = self.internal_storage_method self._image_storage = self.image_storage self._audio_storage = self.audio_storage self._label_storage = self.label_storage self._channel = self.channel if self._version < 0.4: self._per_channel_storage_method={} for channel in self._channel.keys(): self._per_channel_storage_method[channel]=self._internal_storage_method class LabeledMovieHeader(object): """ :param fps: frames per second, parameter defining the sampling rate of the movie necessary for proper playback :type fps: float :param created_date: date of creation as returned by time.ctime() :type created_date: str :param author: Author information :type author: str :param copyright: Copyright information :type copyright: str A header object containing additional information about the collection of movie frames stored in a file. """ def __init__(self, fps=20.0, created_date=time.ctime(), author="", copyright="2015 Brain Corporation"): self._fps = fps self._version = 0.1 self._created_date = created_date self._author = author self._copyright = copyright @property def fps(self): """ Get the frame per second parameter :return: fps :rtype: float """ return self._fps @property def version(self): """ Get the version parameter :return: version :rtype: float """ return self._version @property def created_date(self): """ Get the creation date parameter :return: date as returned by time.ctime() :rtype: str """ return self._created_date @property def author(self): """ Get the author information :return: Author information string :rtype: str """ return self._author @property def copyright(self): """ Get the copyright information :return: copyright :rtype: str """ return self._copyright class LabeledMovieWriter(object): """ :param filename: name of the file to written. Caution, if file exists it will be overwritten :type filename: str :param movie_header: Additional movie information header :type movie_header: PVM_tools.labeled_movie.LabeledMovieHeader A helper class allowing to write frames to a file. If the movie_header argument is not given it will generate a default LabeledMovieHeader object. :Example: :: cap = cv2.VideoCapture(-1) wr = LabeledMovieWriter("./movie_test.pkl") for i in xrange(100): ret, img = cap.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.set_image(img) wr.write_frame(F) wr.finish() """ def __init__(self, filename, movie_header=LabeledMovieHeader()): self.file = open(filename, mode="wb") self._movie_header = movie_header self._movie_header_written = False def write_frame(self, Frame): """ :param Frame: a frame to be written :type Frame: PVM_tools.labeled_movie.LabeledMovieFrame """ if not self._movie_header_written: cPickle.dump(obj=self._movie_header, file=self.file, protocol=-1) self._movie_header_written = True cPickle.dump(obj=Frame, file=self.file, protocol=-1) def finish(self): """ Finish a session and close the file. """ self.file.close() class LabeledMovieReader(object): """ :param filename: path to the file :type filename: str A helper class for reading a collection of frames from a file. """ def __init__(self, filename): self.file = open(filename, mode="rb") self._has_frames = True try: self._header = CoreUtils.load_legacy_pickle_file(self.file) except EOFError: self._has_frames = False @property def fps(self): """ Property returning the frame per second parameter from the file header :return: """ if self._has_frames: return self._header.fps else: return None def frames(self): """ A generator call for returning the collection of frames. :Example: :: mr = LabeledMovieReader("./movie_test.pkl") for Frame in mr.frames(): img = Frame.get_image() cv2.imshow("Image", img) cv2.waitKey(int(1000/mr.fps)) :return: a frame iterator """ ReferenceFrame = LabeledMovieFrame() ver = ReferenceFrame.version warning_message = True while True: try: F = CoreUtils.load_legacy_pickle_file(self.file) if F.version < ver: if warning_message: print "This movie uses old version frames (%2.2f), needs upgrading" % F.version print "Performance will be affected!" warning_message = False F = LabeledMovieFrame.upgrade_to_latest_version(F) yield F except EOFError: self._has_frames = False break def get_header(self): """ Returns the entire header object """ return self._header class FrameCollection(object): """ A helper class encapsulating some of the details of working with a collection of frames. Encapsulates the movie header object, a list of frames and provides convenient read and write mechanisms. :Example: :: fc = FrameCollection() for i in xrange(100): ret, img = cap.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.set_image(img) fc.append(F) fc.reverse() for image in fc: cv2.imshow("frame", image) cv2.waitKey(int(1000/fc.fps)) """ def __init__(self, channel="default", movie_header=LabeledMovieHeader(fps=20)): self._list = [] self._channel = channel self._header = movie_header def append(self, Frame): """ Add a frame to the collection :param Frame: frame object :type Frame: PVM_tools.labeled_movie.LabeledMovieFrame """ self._list.append(Frame) def set_active_channel(self, channel): """ :param channel: channel to select :type channel: str :return: Selects the active channel from which image data can be conveniently extracted by list interface """ self._channel = channel def delete(self, Frame): """ :param Frame: index of, or the frame object itself to be removed :type Frame: int|PVM_tools.labeled_movie.LabeledMovieFrame Removes a frame from the collection """ if type(Frame) == int: self._list.pop(Frame) elif type(Frame) == LabeledMovieFrame: self._list.remove(Frame) def __getitem__(self, item): self._list[item].get_image(channel=self._channel) def __iter__(self): for F in self._list: yield F.get_image(channel=self._channel) def __len__(self): return len(self._list) def reverse(self): """ Reverses the order of frames :return: """ self._list.reverse() @property def fps(self): """ Get the frame per second property :return: """ return self._header.fps def write_to_file(self, filename): """ Use a LabeledMovieWriter class to write the entire collection to a file. :param filename: path to the file written :type filename: str """ mw = LabeledMovieWriter(filename=filename, movie_header=self._header) for Frame in self._list: mw.write_frame(Frame) mw.finish() def load_from_file(self, filename): """ :param filename: path to the file read :type filename: str Load a collection from a file using the LabeledMovieReader class """ mr = LabeledMovieReader(filename=filename) self._header = mr.get_header() self._list = [] for Frame in mr.frames(): self._list.append(Frame) def Frame(self, index): """ :param index: Index of the frame to be exposed :type index: int :return: PVM_tools.labeled_movie.LabeledMovieFrame Exposes the frame in the collection at a given index. """ return self._list[index] if __name__ == "__main__": cap = cv2.VideoCapture(-1) wr = LabeledMovieWriter("./movie_test.pkl") fc = FrameCollection() for i in xrange(100): ret, img = cap.read() F = LabeledMovieFrame(internal_storage_method="jpg") F.set_image(img) wr.write_frame(F) fc.append(F) wr.finish() fc.reverse() for image in fc: cv2.imshow("frame", image) cv2.waitKey(int(1000/fc.fps)) mr = LabeledMovieReader("./movie_test.pkl") for Frame in mr.frames(): img = Frame.get_image() cv2.imshow("Image", img) cv2.waitKey(int(1000/mr.fps)) ================================================ FILE: README.md ================================================ # Predictive Vision Model ## Introduction This repository contains necessary files and demos used for running multicore simulations for the Predictive Vision Model paper entitled *"Unsupervised Learning from Continuous Video in a Scalable Predictive Recurrent Network"* (Piekniewski et al., 2016 (https://arxiv.org/abs/1607.06854)). The code includes a framework that can run a large number of python objects in parallel, synchronized by global barriers, utilizing state kept in shared memory. ## Starting up For the quickest setup get a clean Ubuntu 16.04 machine with as many compute cores as you can get (should also work fine on 15.10, 15.04 and 14.04 but 16.04 is tested). Next clone the repo and run the following commands: ``` git clone git@github.com:braincorp/PVM cd PVM sudo ./install_ubuntu_dependecies.sh ``` This script will install all the necessary packages like opencv, numpy, gcc, cython etc. This may take a while depending on your internet connection. Once this is done, you have to compile a few things and initialize github modules. To do that, run: ``` source install_local.sh ``` This among other things will compile Cython/Boost bindings. Make sure there are no error messages before you proceed. The script above will also update the PYTHONPATH variable so that you can now run the content from the current terminal window. Note that if you open another window you will have to set the PYTHONPATH. [Documentation](http://pvm.braincorporation.net/docs/index.html) for the project is available online. ### Structure of the repo There are several packages available in the repo. Since this code is meant for experimentation, they are not very strictly separated: * PVM_framework - classes and functions necessary to instantiate and run multicore simulations * PVM_models - demos and PVM run scripts, where you will spend most of the time initially * PVM_tools - classes and functions related to labeling and storing labeled movies, tracker benchmark functions * other_trackers - several git submodules and python binding for state of the art trackers. Included are CMT, OpenTLD, STRUCK, a simple color tracker and stationary control trackers * tracker_tools - additional scripts for importing/exporting/labeling/playing the tracking datasets * docs - is where you can generate the documentation using sphinx (just run the ```generate.sh``` script) ## Download data In order to run some of the demos and models you will need data. Please download the following files and unzip the contents into the HOME directory: ``` cd wget http://pvm.braincorporation.net/PVM_data_sequences.zip unzip PVM_data_sequences.zip rm PVM_data_sequences.zip ``` If you are interested in replicating the results from Piekniewski et al., 2016 (https://arxiv.org/abs/1607.06854), you may want to download a few pre-trained models: ``` cd wget http://pvm.braincorporation.net/PVM_data_models01.zip unzip PVM_data_models01.zip rm PVM_data_models01.zip ``` You can also download the data and the models automatically by running the following script and answering 'yes' to each prompt: ``` ./download_data.sh ``` Note that the files range from 1.5GB to 3.5GB totalling up to 7GB of drive space necessary. Make sure to have enough space available before you proceed. The above commands will create a directory subtree PVM_data in your home directory. PVM will load and save a bunch of data in that structure. In addition by configuring the PVM_Storage.py and setting Amazon S3 credentials in ~/.aws/credentials file, PVM can use an S3 bucket to mirror the data directory. This is useful when running simulations on a cluster etc. ## Basic demos In order to familiarize yourself with this framework you may want to run/read several small demos. Running them is not necessary for reproducing the results of the paper, but could be fun on its own. If you are more interested with the PVM itself, it is safe to skip to the next section. Otherwise go to PVM_models directory and run: ``` python demo00_run.py ``` A window full of flickering black and white pixels should appear. Depending on you machine speed the pixels will update randomly. Check your CPU usage with top or activity monitor. It should be high. Next run: ``` python demo01_run.py ``` This is a bit more interesting as this demo will simulate the Ising model at critical temperature. You may decide to look at it for a while, as interesting structures may emerge. [ESC] will quit. Demos 2,3,4 require a camera or an input movie. These demos are more along the lines of what we will see with the full PVM simulation (however we provide the data files for PVM). Unless a movie is given through -f parameter the demos will try to open a camera. ``` python demo02_run.py -f my_movie_file.avi python demo03_run.py -f my_movie_file.avi python demo04_run.py -f my_movie_file.avi ``` where ```my_movie_file.avi``` is any movie file which you have to supply. Demo 2 is a tile of small predictive encoders trying to predict a movie frame based on the previous two frames. In Demo 3, an image is being predicted by a set of predictive encoders that look at different aspects of the input. One set of these "future encoders" digests two 8x8 frames to predict the next one, another set taking 8 4x4 frames to predict 4 subsequent frames and another set taking 32 2x2 frames to predict 16 subsequent frames. Additionally there is one unit that takes the whole image as 8x8 block whose internal representations are shared as context with all the other units. The system has feedback connections, those units with larger temporal buffers feed back to those with more spatial information. Also a cross-like neighbourhood of lateral projections is instantiated. In Demo 4, a future/predictive encoder is instantiated that predicts a camera image based on two previous frames. Here, the encoder predicts the signal and its own error for that signal (i.e., an additional set of units tries to predict the magnitude of error between the prediction and the signal). In addition the hidden layer activations from the previous step of execution are used as as context input block. Second order error is calculated as the error of the error prediction. Also, the learning rate of the primary signal is modulated by the magnitude of the second order error. ## The PVM model The PVM is a larger project that to some extent resembles Demos 2-4, but is much more complex. It constructs a hierarchy of predictive encoders as described in the paper *[Unsupervised Learning from Continuous Video in a Scalable Predictive Recurrent Network]* (https://arxiv.org/abs/1607.06854). The model may be evaluated on online visual tracking task. The descriptions of meta-parameters of the models that can be simulated are available in ```PVM_models/model_zoo```. Feel free to modify these settings, but be aware that simulating some of these models to convergence may take days if not weeks (at least until a GPU version is implemented, which is not yet the case). For your convenience several pre-trained models are available. ### PVM data storage In general all training data and models for PVM are stored in the ```~/PVM_data``` directory. If you have amazon credentials and an S3 bucket then ```~/PVM_data``` can be mirrored to that bucket. This is useful when running models on the Amazon Web Services EC2 (Elastic Compute Cloud) on a larger scale. ### Running PVM Once the data and code is in place you may run one of the already pre-trained instances e.g.: ``` cd PVM/PVM_models python PVM_run.py -D -c 10 -r PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0095000000.p.gz -s stop_sign ``` This will load and run the simulation stored in "PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0095000000.p.gz" utilizing 10 compute cores (-c 10), on the stop sign data set training files (-s) and will display a live preview window (-D). Live preview is good for tinkering but may slow down the execution. ESC pressed on the window will finish the execution. ### Debug interface When PVM_run.py executes it will typically print out several messages, one of which may look like this: ``` Listening on port 9000 for debug connections ``` This indicates a port on which a debug server is listening. Debug shell is a powerful tool that allows for various experiments on a running model. Invoke the shell by logging into the port, e.g. using netcat: ``` nc localhost 9000 ``` Now you are in the debug shell. Type "help" for list of possible commands and "help command" for help on particular command. Using debug shell you can pause and step simulation, freeze and unfreeze learning, modify parameters and even execute full blown python commands. The simulation is presented in the shell as a file system that you can traverse just like an ordinary Unix filesystem. Some elements are leafs of the dictionary (which would correspond to files). You can display their contents via "cat". Others that are subdictionaries or sublists correspond to dictionaries that you can traverse with "cd". Importantly, while connected to the debug console, you can enable/disable the window preview of the running simulation via: ``` toggle_display ``` This window allows you to see what is going on, but it will slow down the simulation a bit, therefore we recommend disabling it for long bouts of training. #### Leftover processes This code is just for experiments and not for production so things may sometimes go wrong and the simulation either crashes or you have to Ctrl+C it. In such situation some of the compute threads may keep running on the CPU in a busy loop and substantially decrease the performance of your machine. It is recommended that you verify what is running every once in a while with the top command. To kill processes selectively you may use commands like this: ``` ps -ef | grep PVM_run | cut -f 2 -d " " | xargs kill ``` This will kill every process that contains ```PVM_run``` in its command line. By tweaking the grep string you may be more or less selective about what you will kill. Be careful, since this command may accidentally kill the processes you did not intend to kill! Make sure the grep selection sieves out only those proceeses that you actually need removed! ### Tracker benchmark The PVM package comes with infrastructure for evaluating tracking performance. The provided data files are divided into essentially three groups (check PVM_datasets.py for details): * Training data (selected by adding suffix ```"_training"``` to dataset name) * Testing data (selected by adding suffix ```"_testing"``` to dataset name) * Extended testing data (selected by adding suffix ```"_ex_testing"``` to dataset name) The selection is arbitrary and you are free to modify those assignments. In addition both testing sets can ebe combined by selecting suffix ```"_full_testing"```. You can run benchmark by invoking the command in ```PVM_models```, e.g: ``` python run_tracking_benchmark.py -T0 -T1 -T2 -s stop_sign_full -r PVM_models/ 2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0095000000.p.gz -c 5 -S 2 -e -p test ``` This will run the PVM tracker (-T0) along with null and center trackers (-T1, -T2) on ```stop_sign_full``` set, the PVM tracker will run two steps on every frame to allow its dynamics to settle a bit (-S 2), the results will be saved in ```~/benchmark_results/test[...]```. The -e flag is required to actually **execute** the benchmark, otherwise the script will try to recompute the results saved from previous runs. The full set of options is available by running the script without any parameters, i.e.: ``` python run_tracking_benchmark.py ``` ### Reproducing paper results The paper *"Unsupervised Learning from Continuous Video in a Scalable Predictive Recurrent Network"* (Piekniewski et al., 2016)(https://arxiv.org/abs/1607.06854) contains three experiments. The software included here allows for reproduction of two of them. The third one involves an integration with virtual world (Unreal 4.9), which we are currently not releasing, but may release in the future. #### Experiment 1 Experiment 1 can be reproduced from scratch (i.e., training) or from a set of pre-trained models that we release along with the code. To reproduce from scratch run the following commands: ``` python PVM_run.py -B -S model_zoo/experiment1_green_ball.json python PVM_run.py -B -S model_zoo/experiment1_stop_sign.json python PVM_run.py -B -S model_zoo/experiment1_face.json ``` We recommend to run each of these commands on a separate, powerful machine. Now all you need to do is wait. Depending on the available CPU resources, it will take 10h-30h to simulate 1M steps. To reproduce the data presented in the paper you need to run for 95M steps, which might be a while (although you should get reasonable performance after 20M though). As the simulation proceeds it will write snapshots every 100k steps, so make sure there is enough drive space available as well. Each snapshot may take 200MB. After this is completed you may directly run the benchmark on these snapshots as here: ``` python run_tracking_benchmark.py -T0 -T1 -T2 -s stop_sign_full -r PVM_models/ PATH_TO_MY_SNAPSHOT/PVM_failsafe_0095000000.p.gz -c 5 -S 4 -e -p test ``` If you dont want to wait several months of training then you can use the pre trained models we ship with the code. Download and unzip the file ```PVM_data_models01.zip```. The file contains 9 files: ``` drwxrwxr-x 3.0 unx 0 bx stor 16-Jun-23 10:49 PVM_data/PVM_models/2016_01_08_18_56_56_face_simple____47aaabd4/ -rw-rw-r-- 3.0 unx 187379490 bx defN 16-Jun-23 10:48 PVM_data/PVM_models/2016_01_08_18_56_56_face_simple____47aaabd4/PVM_failsafe_0040000000.p.gz -rw-rw-r-- 3.0 unx 189374856 bx defN 16-Jun-23 10:45 PVM_data/PVM_models/2016_01_08_18_56_56_face_simple____47aaabd4/PVM_failsafe_0095000000.p.gz -rw-rw-r-- 3.0 unx 186499761 bx defN 16-Jun-23 10:49 PVM_data/PVM_models/2016_01_08_18_56_56_face_simple____47aaabd4/PVM_failsafe_0020000000.p.gz drwxrwxr-x 3.0 unx 0 bx stor 16-Jun-23 10:19 PVM_data/PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/ -rw-rw-r-- 3.0 unx 186445343 bx defN 16-Jun-23 10:44 PVM_data/PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0020000000.p.gz -rw-rw-r-- 3.0 unx 187455605 bx defN 16-Jun-23 10:43 PVM_data/PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0040000000.p.gz -rw-rw-r-- 3.0 unx 189399162 bx defN 16-Jun-23 10:40 PVM_data/PVM_models/2016_01_08_18_54_44_stop_simple____dd25d826/PVM_failsafe_0095000000.p.gz drwxrwxr-x 3.0 unx 0 bx stor 16-Jun-23 10:13 PVM_data/PVM_models/2016_01_08_19_01_50_green_b_simple_d547417c/ -rw-rw-r-- 3.0 unx 186473825 bx defN 16-Jun-23 10:39 PVM_data/PVM_models/2016_01_08_19_01_50_green_b_simple_d547417c/PVM_failsafe_0020000000.p.gz -rw-rw-r-- 3.0 unx 187426899 bx defN 16-Jun-23 10:37 PVM_data/PVM_models/2016_01_08_19_01_50_green_b_simple_d547417c/PVM_failsafe_0040000000.p.gz -rw-rw-r-- 3.0 unx 189394630 bx defN 16-Jun-23 10:36 PVM_data/PVM_models/2016_01_08_19_01_50_green_b_simple_d547417c/PVM_failsafe_0095000000.p.gz ``` Pass these to the tracking benchmark to reproduce the majority of the results. #### Experiment 2 In this case the model is trained in an unsupervised way for a long time. Much like in the Experiment 1 you can train a model from scratch or use one of our pre-trained instances. We constructed a dataset out of clips from all the three categories plus some movies without any target labeled. The dataset is called "non_spec" to indicate there is no specific target in it. To train the model run: ``` python PVM_run.py -B -S model_zoo/experiment2.json ``` And wait. Much like in Experiment 1 it will take weeks if not months to get to 100M steps. For the pre-trained files, download and unzip ```PVM_data_models03.zip```. The zip contains 5 files: ``` -rw-rw-r-- 3.0 unx 187227041 bx defN 16-Jun-23 11:31 PVM_data/PVM_models/2016_03_21_17_03_18_primable_simple_polynomial_small_11e4cc9f/PVM_failsafe_0020000000.p.gz -rw-rw-r-- 3.0 unx 187648051 bx defN 16-Jun-23 11:30 PVM_data/PVM_models/2016_03_21_17_03_18_primable_simple_polynomial_small_11e4cc9f/PVM_failsafe_0040000000.p.gz -rw-rw-r-- 3.0 unx 190867620 bx defN 16-Jun-23 11:28 PVM_data/PVM_models/2016_03_21_17_03_18_primable_simple_polynomial_small_11e4cc9f/PVM_failsafe_0095000000.p.gz drwxrwxr-x 3.0 unx 0 bx stor 16-Jun-23 11:44 PVM_data/PVM_models/2016_02_10_02_16_33_primable_simple_large_52ae3b91/ -rw-rw-r-- 3.0 unx 679544447 bx defN 16-Jun-23 11:44 PVM_data/PVM_models/2016_02_10_02_16_33_primable_simple_large_52ae3b91/PVM_failsafe_0020000000.p.gz -rw-rw-r-- 3.0 unx 680852880 bx defN 16-Jun-23 11:35 PVM_data/PVM_models/2016_02_10_02_16_33_primable_simple_large_52ae3b91/PVM_failsafe_0040000000.p.gz ``` The "primable_simple_polynomial_small" snapshots are instances of the models presented in the paper. The "primable_simple_large" is a similar instance but much bigger (and slower). We have included the bigger instance for completeness, but it is not used for any of the results presented in the paper. Once the unsupervised model is pre-trained, you can now start the second phase of training, in supervised mode. To do that, run: ``` python PVM_run.py -r PATH_TO/pretrained_model.p.gz -D -O '{"supervised": "1", "supervised_rate": "0.0002", "dataset": "stop_sign", "steps": "10000" }' ``` Modify the parameters to fit the needs of your experiment. For one of our pre-trained models this command looks like this (recall you may skip the display option ```-D``` for faster run time): ``` python PVM_run.py -r PVM_models/2016_03_21_17_03_18_primable_simple_polynomial_small_11e4cc9f/PVM_failsafe_0095000000.p.gz -D -O '{"supervised": "1", "supervised_rate": "0.0002", "dataset": "stop_sign", "steps": "100000" }' ``` ## Summary The software presented here allows for reproduction of the majority of tracking results presented in the paper *"Unsupervised Learning from Continuous Video in a Scalable Predictive Recurrent Network"* (Piekniewski et al., 2016) (https://arxiv.org/abs/1607.06854). This code is meant for scientific experimental purposes and should not be considered production-ready software (see also the license restrictions preventing you from using this code for commercial purposes). Brain Corporation provides no warranty about functionality of this code, and you are using it at your own risk. In particular, these simulations are compute-intensive and we are not responsible if they cause your computer to run at high temperatures and/or cause excessive CPU heating. You can and should read all the source provided here to make sure you know what this code is doing (hopefully it is reasonably documented). We recommend also reading the paper. Enjoy! P.S.: Contact us if you wish to contribute. ================================================ FILE: add_thirdparty_tracker_submodules.sh ================================================ git submodule add https://github.com/gnebehay/STRUCK other_trackers/original_struck cd other_trackers/original_struck git checkout d6facc164385013f69e7f9e910e4167ee020f922 cd ../../ git submodule add https://github.com/gnebehay/OpenTLD other_trackers/original_opentld cd other_trackers/original_opentld git checkout 82631ed299dbf8227d8c4b844e3227adcb165cb5 cd ../../ git submodule add https://github.com/gnebehay/CMT other_trackers/original_cmt cd other_trackers/original_cmt git checkout fd936ec8616f9036d520f35e79638e448e233582 cd ../../ ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PredictiveVisionFramework.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PredictiveVisionFramework.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/PredictiveVisionFramework" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PredictiveVisionFramework" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # Predictive Vision Framework documentation build configuration file, created by # sphinx-quickstart on Wed Jun 22 17:54:18 2016. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Predictive Vision Framework' copyright = u'2016, Brain Corporation' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.0' # The full version, including alpha/beta/rc tags. release = '1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'PredictiveVisionFrameworkdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'PredictiveVisionFramework.tex', u'Predictive Vision Framework Documentation', u'Filip Piekniewski', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'predictivevisionframework', u'Predictive Vision Framework Documentation', [u'Filip Piekniewski'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'PredictiveVisionFramework', u'Predictive Vision Framework Documentation', u'Filip Piekniewski', 'PredictiveVisionFramework', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} ================================================ FILE: docs/generate.sh ================================================ sphinx-apidoc -o . ../ -e make html ================================================ FILE: docs/index.rst ================================================ .. Predictive Vision Framework documentation master file, created by sphinx-quickstart on Wed Jun 22 17:54:18 2016. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to Predictive Vision Framework's documentation! ======================================================= Contents: .. toctree:: :maxdepth: 2 Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/make.bat ================================================ @ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . set I18NSPHINXOPTS=%SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. xml to make Docutils-native XML files echo. pseudoxml to make pseudoxml-XML files for display purposes echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) %SPHINXBUILD% 2> nul if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PredictiveVisionFramework.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PredictiveVisionFramework.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdf" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdfja" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf-ja cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) if "%1" == "xml" ( %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml if errorlevel 1 exit /b 1 echo. echo.Build finished. The XML files are in %BUILDDIR%/xml. goto end ) if "%1" == "pseudoxml" ( %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml if errorlevel 1 exit /b 1 echo. echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. goto end ) :end ================================================ FILE: download_data.sh ================================================ cd echo "Downloading the labeled movie sequences (3GB). Be patient." wget http://pvm.braincorporation.net/PVM_data_sequences.zip unzip PVM_data_sequences.zip rm PVM_data_sequences.zip echo "Would you like to download the first set of pre-trained models? (1.6GB)" read -r -p "Are you sure? [Y/n]" response response=${response,,} if [[ $response =~ ^(yes|y| ) ]]; then wget http://pvm.braincorporation.net/PVM_data_models01.zip unzip PVM_data_models01.zip rm PVM_data_models01.zip fi echo "Would you like to download the second set of pre-trained models? (2.7GB)" read -r -p "Are you sure? [Y/n]" response response=${response,,} if [[ $response =~ ^(yes|y| ) ]]; then wget http://pvm.braincorporation.net/PVM_data_models02.zip unzip PVM_data_models02.zip rm PVM_data_models02.zip fi echo "Would you like to download the third set of pre-trained models? (1.8GB)" read -r -p "Are you sure? [Y/n]" response response=${response,,} if [[ $response =~ ^(yes|y| ) ]]; then wget http://pvm.braincorporation.net/PVM_data_models03.zip unzip PVM_data_models03.zip rm PVM_data_models03.zip fi cd - ================================================ FILE: install_local.sh ================================================ git submodule init git submodule update python setup.py build_ext --inplace export PYTHONPATH="$PYTHONPATH:`pwd`" cd other_trackers python setup_struck.py build_ext --inplace python setup_opentld.py build_ext --inplace cd - ================================================ FILE: install_ubuntu_dependencies.sh ================================================ #!/bin/bash apt-get -y install git apt-get -y install wget apt-get -y install python-opencv apt-get -y install python-numpy apt-get -y install python-posix-ipc apt-get -y install python-pip apt-get -y install python-virtualenv apt-get -y install python-matplotlib apt-get -y install python-boto apt-get -y install cython apt-get -y install python-scipy apt-get -y install python-sphinx apt-get -y install python-pytest apt-get -y install libcv-dev apt-get -y install libhighgui-dev apt-get -y install libboost-python-dev apt-get -y install gcc apt-get -y install g++ apt-get -y install libeigen3-dev apt-get -y install libopencv-dev ================================================ FILE: other_trackers/ASMSearcher.hpp ================================================ /** # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== **/ #ifndef ASMSearcher_HPP #define ASMSearcher_HPP #include #include #include struct cvmat_t { PyObject_HEAD CvMat *a; PyObject *data; size_t offset; }; struct iplimage_t { PyObject_HEAD IplImage *a; PyObject *data; size_t offset; }; cv::Mat convert_from_cvmat(PyObject *o, const char* name) { cv::Mat dest; cvmat_t *m = (cvmat_t*) o; void *buffer; Py_ssize_t buffer_len; m->a->refcount = NULL; if (m->data && PyString_Check(m->data)) { assert(cvGetErrStatus() == 0); char *ptr = PyString_AsString(m->data) + m->offset; cvSetData(m->a, ptr, m->a->step); assert(cvGetErrStatus() == 0); dest = m->a; } else if (m->data && PyObject_AsWriteBuffer(m->data, &buffer, &buffer_len) == 0) { cvSetData(m->a, (void*) ((char*) buffer + m->offset), m->a->step); assert(cvGetErrStatus() == 0); dest = m->a; } else { printf("CvMat argument '%s' has no data", name); } return dest; } cv::Mat convert_from_cviplimage(PyObject *o, const char *name) { cv::Mat dest; iplimage_t *ipl = (iplimage_t*) o; void *buffer; Py_ssize_t buffer_len; if (PyString_Check(ipl->data)) { cvSetData(ipl->a, PyString_AsString(ipl->data) + ipl->offset, ipl->a->widthStep); assert(cvGetErrStatus() == 0); dest = ipl->a; } else if (ipl->data && PyObject_AsWriteBuffer(ipl->data, &buffer, &buffer_len) == 0) { cvSetData(ipl->a, (void*) ((char*) buffer + ipl->offset), ipl->a->widthStep); assert(cvGetErrStatus() == 0); dest = ipl->a; } else { printf("IplImage argument '%s' has no data", name); } return dest; } cv::Mat convertObj2Mat(boost::python::object image) { if (strcmp(image.ptr()->ob_type->tp_name, "cv2.cv.iplimage") == 0) { return convert_from_cviplimage(image.ptr(), image.ptr()->ob_type->tp_name); } else return convert_from_cvmat(image.ptr(), image.ptr()->ob_type->tp_name); } #endif ================================================ FILE: other_trackers/__init__.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== ================================================ FILE: other_trackers/backprojection.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import cv2 class ColorHistogramBackProjection(object): ''' Takes a color histogram from a sample image and "back-projects" the histogram, using it as a color filter for the image. Outputs a float image in range (0, 1), indicating membership to color histogram. :param n_bins: number of bins on each channel :type n_bins: 3-tuple :param channels: the image channels to draw the historgram from. :type channels: 3-tuple :param ranges: min and max for each channel :type ranges: 3-tuple :param use_background: if True, the histogram values of the target bounding box will be divided by the whole image histogram values, so that the backprojection values represent p(target|bin) :type use_background: Boolean .. automethod:: _update_hist ''' def __init__(self, n_bins=(16, 16, 16), channels=(0, 1, 2), ranges=(0, 256, 0, 256, 0, 256), use_background=True): self.n_bins = n_bins self._ranges = ranges self._channels = channels self.use_background = use_background self.hist = None self.raw_hist_target = None self.raw_hist_image = None self.out = None def calculateHistogram(self, image, bb=None): ''' :param image: input image :type image: numpy.ndarray :param bb: bounding box :type bb: numpy.ndarray If bb (=(x, y, w, h)) is not None, the histogram is taken from the bounded part of the image. If use_background is True, bb cannot be None .. note:: The bounding box here is given in pixel coordinates! ''' assert not (self.use_background and bb is None), 'If using background, bb must be provided' if bb is not None: x, y, w, h = bb target = image[y:y + h, x:x + w] else: target = image hist_target = cv2.calcHist([target], channels=self._channels, mask=None, histSize=self.n_bins, ranges=self._ranges) if self.raw_hist_target is None: self.raw_hist_target = hist_target else: self.raw_hist_target += hist_target if self.use_background: hist_image = cv2.calcHist([image], channels=self._channels, mask=None, histSize=self.n_bins, ranges=self._ranges) if self.raw_hist_image is None: self.raw_hist_image = hist_image else: self.raw_hist_image += hist_image self._update_hist() def _update_hist(self): """ Internal method to calculate the actual histogram """ if self.use_background: if self.raw_hist_image is not None: self.hist = self.raw_hist_target / (self.raw_hist_image + 1.) else: self.hist = self.raw_hist_target.copy() else: self.hist = cv2.normalize(self.raw_hist_target, alpha=0., beta=1., norm_type=cv2.NORM_MINMAX) def get_color_histogram(self): """ Returns the current histogram :return: current histogram :rtype: numpy.ndarray of type float32 """ return self.hist def add_color_histogram(self, hist): """ Additive modification of the current histogram :param hist: a numpy array in the same shape as the current histogram :type hist: numpy.ndarray """ if self.raw_hist_target is None: self.raw_hist_target = hist.copy() else: self.raw_hist_target += hist self._update_hist() def subtract_color_histogram(self, hist): """ Subtractive modification of the current histogram :param hist: a numpy array in the same shape as the current histogram :type hist: numpy.ndarray """ if self.raw_hist_target is not None: self.raw_hist_target -= hist self.raw_hist_target[self.raw_hist_target < 0] = 0 self._update_hist() def reset(self): """ Reset the object """ self.raw_hist_target = None self.raw_hist_image = None self.hist = None def calculate(self, image): """ Calculate the histogram packprojection on the given image using the currently stored histogram :param image: numpy array with appropriate number of channels :type image: numpy.ndarray :return: a numpy array of type float32 containing the histogram backprojection :rtype: numpy.ndarray of type float32 """ if self.hist is None: return np.zeros(image.shape[:-1], dtype=np.float32) out = cv2.calcBackProject([image.astype('float32')], channels=self._channels, hist=self.hist, ranges=self._ranges, scale=1) self.out = out return self.out ================================================ FILE: other_trackers/bounding_boxer.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np import cv2 from PVM_tools.abstract_bounding_boxer import AbstractBoundingBoxer from PVM_tools.bounding_region import BoundingRegion class CamshiftBoundingBoxer(AbstractBoundingBoxer): """ Given a prob-map of pixel membership, return a bounding box: (PVM_tools.bounding_region.BoundingRegion) This class uses the opencv CamShift method to estimate the new position of the bounding box based on the previous one. In the case the previous bounding box is empty, then a search over the entire map is performed to estimate the new position. """ def __init__(self, recovery_kernel_size=20, min_recovered_box_area=400, threshold_retrieve=0.2): """ :param recovery_kernel_size: when searching for a new box, the probablity map will be filtered with a box filter, to smooth out single peaks and find a better candidate. This is the size of the box kernal. :type recovery_kernel_size: int :param min_recovered_box_area: after finding a box, assert that width x height is at least this number of pixels. Otherwise do not accept that as a good box and return an empty box. :type min_recovered_box_area: int :param threshold_retrieve: When object the is lost, that is the minimal confidence the tracker must get to consider that a new candidate is considered as the new target Lower value (ex 0.1) provides better recovery but more false positive. Typical value are between 0.1 and 0.2 :type threshold_retrieve: float """ self.term_crit = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1) self._last_bb = None self.confidence = 1. self.recovery_kernel_size = recovery_kernel_size self.min_recovered_box_area = min_recovered_box_area self.threshold_retrieve = threshold_retrieve def reset(self): """ Reset the object. """ self._last_bb = None def _find_new_box(self, heatmap): heatmap = cv2.boxFilter(heatmap, ddepth=-1, ksize=(self.recovery_kernel_size, self.recovery_kernel_size), borderType=cv2.BORDER_REPLICATE) # new_bb = np.array(np.unravel_index(np.argmax(conv_prob), conv_prob.shape))[::-1] # new_bb = np.array(tuple(new_bb - max(1, min(10, self.recovery_kernel_size/2))) + (min(20, self.recovery_kernel_size), ) * 2) # _, bb_reset = cv2.CamShift(heatmap, tuple(new_bb), self.term_crit) peak = np.unravel_index(np.argmax(heatmap), heatmap.shape) _, heatmap = cv2.threshold(heatmap, np.max(heatmap)*0.9, 255, cv2.THRESH_BINARY) _, bb_reset = cv2.floodFill(heatmap, None, tuple(peak[::-1]), 255, loDiff=10, flags=cv2.FLOODFILL_FIXED_RANGE) bb_area = bb_reset[2] * bb_reset[3] mean_prob_reset = np.sum(heatmap[bb_reset[1]:bb_reset[1] + bb_reset[3], bb_reset[0]:bb_reset[0] + bb_reset[2]]) / float(bb_area) return bb_reset, bb_area, mean_prob_reset def process(self, heatmap): """ The main processing method. Given the probability map, returns a bounding region :param heatmap: the map (e.g. result of histogram backprojection etc.). Its supposed to be a numpy array of float. :type heatmap: numpy.ndarray :return: bounding_box :rtype: PVM_tools.bounding_region.BoundingRegion """ self._last_prob = heatmap harea = np.prod(heatmap.shape) if self._last_bb is None or self._last_bb.empty: (bb, area, mean_prob) = self._find_new_box(heatmap) confidence = mean_prob if bb[2] > 0 and bb[3] > 0 and np.prod(bb[2:])<2*harea/3: self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3), box=np.array(bb, dtype=np.int), confidence=confidence) else: # Empty bounding box self._last_bb = BoundingRegion() else: # Step 1 go with the last bbox _, bb0 = cv2.meanShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit) area0 = np.prod(bb0[2:]) mean_prob0 = np.sum(heatmap[bb0[1]:bb0[1] + bb0[3], bb0[0]:bb0[0] + bb0[2]]) / (1e-12 + area0) _, bb = cv2.CamShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit) area = np.prod(bb[2:]) mean_prob = np.sum(heatmap[bb[1]:bb[1] + bb[3], bb[0]:bb[0] + bb[2]]) / (1e-12 + area) if mean_prob > self.threshold_retrieve and area > self.min_recovered_box_area and area < 2*harea/3: self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3), box=np.array(bb, dtype=np.int), confidence=mean_prob) elif mean_prob0 > self.threshold_retrieve and area0 > self.min_recovered_box_area: # Go with the mean shift box, changing size apparently does no good. return self._last_bb else: # Empty bounding box self._last_bb = BoundingRegion() return self._last_bb def set_current_bounding_box(self, bounding_box): """ If the bounding box is somehow available (e.g. in priming) that values can be passed in here to affect future computation (new boxes will be found in reference to this one) :param bounding_box: A bounding region object :type bounding_box: PVM_tools.bounding_region.BoundingRegion """ self._last_bb = bounding_box class FloodFillBoundingBoxer(AbstractBoundingBoxer): """ Given a prob-map of pixel membership, return a bounding box: (PVM_tools.bounding_region.BoundingRegion) This class uses the opencv FloodFill from the peak of the heatmap to estimate the new position of the bounding box. """ def __init__(self, heatmap_threshold=200, distance_threshold=20): self.heatmap_threshold = heatmap_threshold self.distance_threshold = distance_threshold self.recovery_kernel_size = 20 def reset(self): pass def process(self, heatmap, previous_bb=None, peak=None): """ This method computes a bounding box around the heatmap peak. Arguments: heatmap - target position heat map (may have multiple local maxima). peak - [y, x] of the most likely target position (usually the highest peak of the heatmap). This argument tells compute_bounding_box() which local maximum to choose. returns a bounding box array (x_upper_left,y_upper_left,width,height) or None if it can't be found """ if np.max(heatmap) == 0.0: return BoundingRegion() heatmap = cv2.boxFilter(heatmap, ddepth=-1, ksize=(self.recovery_kernel_size, self.recovery_kernel_size), borderType=cv2.BORDER_REPLICATE) if peak is None: peak = np.unravel_index(np.argmax(heatmap), heatmap.shape) if np.issubdtype(heatmap.dtype, np.float): _, heatmap = cv2.threshold(heatmap, self.heatmap_threshold*(1.0/255), 255, cv2.THRESH_BINARY) else: _, heatmap = cv2.threshold(heatmap, self.heatmap_threshold, 255, cv2.THRESH_BINARY) if heatmap[peak[0], peak[1]] != 255: cors = np.nonzero(heatmap) new_peak = None if len(cors[0]) > 0: dist2 = (cors[0] - peak[0])**2 + (cors[1] - peak[1])**2 ind=np.argmin(dist2) if dist2[ind] < self.distance_threshold**2: new_peak = np.array([cors[0][ind], cors[1][ind]]) if new_peak is None: return BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3), box=np.array([peak[1], peak[0], 1, 1])) peak = new_peak _, bounding_box = cv2.floodFill(heatmap, None, tuple(peak[::-1]), 255, loDiff=10, flags=cv2.FLOODFILL_FIXED_RANGE) return BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3), box=np.asarray(bounding_box)) def set_current_bounding_box(self, bounding_region): pass ================================================ FILE: other_trackers/center_vision_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from PVM_tools.abstract_tracker import GenericVisionTracker from PVM_tools.bounding_region import BoundingRegion import numpy as np class CenterVisionTracker(GenericVisionTracker): """ This class exposes the null vision tracker which just always returns its priming bounding box """ def __init__(self, size_factor=0.2, new_name=None): """ Initialize the tracker """ if new_name is None: self.name = 'center_tracker' else: self.name = new_name self.size_factor = size_factor def reset(self): """ Reset the tracker :return: """ self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ box = np.array([(im.shape[1]-im.shape[1]*self.size_factor)/2, (im.shape[0]-im.shape[0]*self.size_factor)/2, im.shape[1]*self.size_factor, im.shape[0]*self.size_factor]).astype(np.int) self.bbox = BoundingRegion(image_shape=im.shape, box=box) if not self._primed: self._primed = True def _track(self, im): """ Track on given image, rseturn a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again self._primed = False return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.bbox.get_mask() ================================================ FILE: other_trackers/cmt_vision_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from PVM_tools.abstract_tracker import GenericVisionTracker from PVM_tools.bounding_region import BoundingRegion import cv2 # Clone the original CMT tracker code https://github.com/gnebehay/CMT and update the # python path import numpy as np import os import sys class CMTVisionTracker(GenericVisionTracker): """ This class exposes the CMT tracker implemented in http://www.gnebehay.com/CMT/. """ def __init__(self): """ Initialize the tracker """ self.name = 'cmt_tracker' self.reset() def reset(self): """ Reset the tracker :return: """ self.cmt = None self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ self.bbox=bounding_region bounding_box = bounding_region.get_box_pixels() if not self._primed: this_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(this_dir+"/original_cmt/") from CMT import CMT self.cmt = CMT() self._primed = True self.height, self.width = im.shape[:2] im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY) self.cmt.initialise(im_gray0=im, tl=tuple([bounding_box[0], bounding_box[1]]), br=tuple([bounding_box[0]+bounding_box[2], bounding_box[1]+bounding_box[3]])) self.im_prev = im def _track(self, im): """ Track on given image, return a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again self._primed = False self.im_prev = im im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY) self.cmt.process_frame(im) self.confidence = 1 if np.isnan(self.cmt.bb).any(): self.bbox = BoundingRegion() else: self.bbox = BoundingRegion(image_shape=im.shape, box=self.cmt.bb, confidence=self.confidence) return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.bbox.get_mask() ================================================ FILE: other_trackers/color_vision_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import cv2 from PVM_tools.abstract_tracker import GenericVisionTracker from backprojection import ColorHistogramBackProjection from other_trackers.bounding_boxer import CamshiftBoundingBoxer from PVM_tools.bounding_region import BoundingRegion class BasicHistogramBackprojectionTracker(GenericVisionTracker): """ Basic color histogram backprojection tracker as described in [DB90]_. The idea can be summarized as follows: 1. Calculate a color histogram of the image using all or a subset of channels 2. Calculate a color histogram of the region of interest (ROI) 3. Divide the ROI histogram by the whole image histogram (normalization) 4. Backproject the histogram to the image (each pixel of the new heatmap containst the value of the histogram bucket to which the original pixel in the image belonged) 5. Estimate the new position of the target. In this implementation a simple kinematic tracker based on the camshift algorithm [B98]_ is used. This object is a very general tracker that will likely perform very poorly, but it can be tuned by adjusting parameters and selecting appropriate image channels for calculating histograms. In certain cases the histograms can be computed on a highly processed feature representations of the image (e.g. orientations or so on). This module contains a two related trackers that use this basic tracker as a building block. .. [DB90] Swain, M.J.; Ballard, D.H., "Indexing via color histograms," Computer Vision, 1990. Proceedings, Third International Conference on , vol., no., pp.390,393, 4-7 Dec 1990 doi: 10.1109/ICCV.1990.139558 .. [B98] Bradski, G.R., "Real time face and object tracking as a component of a perceptual user interface," Applications of Computer Vision, 1998. WACV '98. Proceedings., Fourth IEEE Workshop on , vol., no., pp.214,219, 19-21 Oct 1998 :param bins: shape of the histogram (number of bins in each channel), default (4, 4, 4) :type bins: tuple :param channels: selection of image channels to be used in histogram calculation, default (0, 1, 2) :type channels: tuple :param ranges: a set of ranges in each channels to which the histogram should be restricted, default (0, 256, 0, 256, 0, 256) :type ranges: tuple :param recovery_kernel_size: refer to :class:`other_trackers.bounding_boxer.CamshiftBoundingBoxer` :type recovery_kernel_size: int :param min_recovered_box_area: refer to :class:`other_trackers.bounding_boxer.CamshiftBoundingBoxer` :type min_recovered_box_area: int :param confidence_threshold_retrieve: refer to :class:`other_trackers.bounding_boxer.CamshiftBoundingBoxer` :type confidence_threshold_retrieve: int :param priming_bbox_rescaling: the factor by which the initial (priming) bounding box should be rescaled before actually priming :type priming_bbox_rescaling: float :param confidence_threshold: confidence level above which a non empty bounding box will be returned :type confidence_threshold: float """ def __init__(self, bins=(4, 4, 4), channels=(0, 1, 2), ranges=(0, 256, 0, 256, 0, 256), recovery_kernel_size=20, min_recovered_box_area=100, confidence_threshold_retrieve=0.2, priming_bbox_rescaling=0.9, confidence_threshold=0.4): self._back_projector = ColorHistogramBackProjection(n_bins=bins, channels=channels, ranges=ranges, use_background=True) self._bounding_boxer = CamshiftBoundingBoxer(recovery_kernel_size=recovery_kernel_size, min_recovered_box_area=min_recovered_box_area, threshold_retrieve=confidence_threshold_retrieve) self._priming_bbox_rescaling = priming_bbox_rescaling self._confidence_threshold = confidence_threshold self._primed = False self.name = "Basic Histogram Backprojection Tracker" def _prime(self, im, bounding_box=None): """ Prime the tracker :param im: priming image :type im: numpy.ndarray :param bounding_box: initial target bounding box :type bounding_box: PVM_tools.bounding_region.BoundingRegion :return: True if primed :rtype: bool """ if bounding_box is None: return if self._priming_bbox_rescaling is not None: bounding_box.scale(self._priming_bbox_rescaling) (x, y, w, h) = bounding_box.get_box_pixels() self._back_projector.calculateHistogram(im, (x, y, w, h)) self.prob = self._back_projector.calculate(im) self._bounding_boxer.set_current_bounding_box(bounding_box) self._primed = True return self._primed def _track(self, im): """ Track :param im: image :type im: numpy.ndarray :return: bounding region :rtype: PVM_tools.bounding_region.BoundingRegion """ if not self._primed: return BoundingRegion() self.prob = self._back_projector.calculate(im) bounding_box = self._bounding_boxer.process(heatmap=self.prob) if bounding_box.confidence > self._confidence_threshold: return bounding_box.copy() else: return BoundingRegion() def reset(self): """ Reset the tracker :return: """ self._back_projector.reset() self._primed = False def get_heatmap(self, heatmap_name=None): """ Get the original histogram backprojection heatmap :param heatmap_name: optional argument for tracker which may have multipe heatmaps :return: heatmap :rtype: numpy.ndarray """ return self.prob class HSHistogramBackprojectionTracker(GenericVisionTracker): """ This class is a variation of the histogram backprojection tracker for the hue and saturation channels. The tracker takes an RGB image, converts it into a proper format and runs through a appropriately configured instance of the backprojection tracker. :param bins: histogram bins shape (8, 8) :type bins: tuple :param channels: selection of channels (0, 1) :type channels: tuple :param ranges: histogram ranges (0, 180, 0, 255) :type ranges: tuple """ def __init__(self): self.tracker = BasicHistogramBackprojectionTracker(bins=(8, 8), channels=(0, 1), ranges=(0, 180, 0, 255)) self.name = "HSTracker" def _preprocess(self, image): """ Preprocess the image :param image: :return: """ return cv2.cvtColor(image, cv2.COLOR_BGR2HSV) def _prime(self, im, bounding_box=None, **kwargs): """ Prime the tracker :param im: priming image :type im: numpy.ndarray :param bounding_box: initial target bounding box :type bounding_box: PVM_tools.bounding_region.BoundingRegion :return: True if primed :rtype: bool """ im = self._preprocess(im) return self.tracker.prime(im, bounding_box=bounding_box) def _track(self, im): """ Track :param im: image :type im: numpy.ndarray :return: bounding region :rtype: PVM_tools.bounding_region.BoundingRegion """ im = self._preprocess(im) return self.tracker.track(im) def reset(self): """ Reset the tracker :return: """ self.tracker.reset() def get_heatmap(self, heatmap_name=None): """ Get the original histogram backprojection heatmap :param heatmap_name: optional argument for tracker which may have multipe heatmaps :return: heatmap :rtype: numpy.ndarray """ return self.tracker.get_heatmap() class UVHistogramBackprojectionTracker(GenericVisionTracker): """ This class is a variation of the histogram backprojection tracker for the U and V chromatic channels of the YUV space. The tracker takes an RGB image, converts it into a proper format and runs through a appropriately configured instance of the backprojection tracker. :param bins: histogram bins shape (8, 8) :type bins: tuple :param channels: selection of channels (0, 1) :type channels: tuple :param ranges: histogram ranges (0, 180, 0, 255) :type ranges: tuple """ def __init__(self): self.tracker = BasicHistogramBackprojectionTracker(bins=(8, 8), channels=(1, 2), ranges=(0, 255, 0, 255)) self.name = "UVTracker" def _preprocess(self, image): """ Preprocess the image :param image: :return: """ return cv2.cvtColor(image, cv2.COLOR_BGR2YUV) def _prime(self, im, bounding_box=None, **kwargs): """ Prime the tracker :param im: priming image :type im: numpy.ndarray :param bounding_box: initial target bounding box :type bounding_box: PVM_tools.bounding_region.BoundingRegion :return: True if primed :rtype: bool """ im = self._preprocess(im) self.tracker.prime(im, bounding_box=bounding_box) def _track(self, im): """ Track :param im: image :type im: numpy.ndarray :return: bounding region :rtype: PVM_tools.bounding_region.BoundingRegion """ im = self._preprocess(im) return self.tracker.track(im) def reset(self): """ Reset the tracker :return: """ self.tracker.reset() def get_heatmap(self, heatmap_name=None): """ Get the original histogram backprojection heatmap :param heatmap_name: optional argument for tracker which may have multipe heatmaps :return: heatmap :rtype: numpy.ndarray """ return self.tracker.get_heatmap() ================================================ FILE: other_trackers/null_vision_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from PVM_tools.abstract_tracker import GenericVisionTracker class NullVisionTracker(GenericVisionTracker): """ This class exposes the null vision tracker which just always returns its priming bounding box """ def __init__(self, scaling=1.0, new_name=None): """ Initialize the tracker """ if new_name is None: self.name = 'null_tracker' else: self.name = new_name self.scaling = scaling def reset(self): """ Reset the tracker :return: """ self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ self.bbox = bounding_region.copy() self.bbox.scale(self.scaling) if not self._primed: self._primed = True def _track(self, im): """ Track on given image, rseturn a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again self._primed = False return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.bbox.get_mask() ================================================ FILE: other_trackers/opentld_python.cpp ================================================ /** # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== **/ #include #include #include #include #include #include #include "TLD.h" #include "DetectorCascade.h" #include "ASMSearcher.hpp" #include #include using namespace boost::python; using namespace cv; using namespace std; namespace tld { class TLD2 { public: TLD2(); // Boost python wrapper functions boost::python::tuple getCurrBB(); void selectObject_numpy(boost::python::object img, const boost::python::tuple bb); void processImage_numpy(boost::python::object img); void set_width_and_height(boost::python::tuple wh); float currConf; TLD* Tracker; cv::Rect bounding_box_; }; TLD2::TLD2() { Tracker = new TLD(); currConf = Tracker->currConf; } void TLD2::selectObject_numpy(boost::python::object img, const boost::python::tuple bb) { bounding_box_ = cv::Rect(extract(bb[0]), extract(bb[1]), extract(bb[2]), extract(bb[3])); cv::Mat image = convertObj2Mat(img); Tracker->selectObject(image, &bounding_box_); } void TLD2::processImage_numpy(boost::python::object img) { cv::Mat image = convertObj2Mat(img); Tracker->processImage(image); currConf = Tracker->currConf; } boost::python::tuple TLD2::getCurrBB() { if (Tracker->currBB != NULL) return make_tuple(Tracker->currBB->x, Tracker->currBB->y, Tracker->currBB->width, Tracker->currBB->height); else return boost::python::tuple(); } void TLD2::set_width_and_height(boost::python::tuple wh){ Tracker->detectorCascade->imgWidth = extract(wh[0]); Tracker->detectorCascade->imgWidthStep = extract(wh[0]); Tracker->detectorCascade->imgHeight = extract(wh[1]); } } BOOST_PYTHON_MODULE(tld) { using namespace boost::python; class_("DetectorCascade", no_init) .def_readwrite("imgWidth", &tld::DetectorCascade::imgWidth) .def_readwrite("imgHeight", &tld::DetectorCascade::imgHeight) .def_readwrite("imgWidthStep", &tld::DetectorCascade::imgWidthStep) ; class_("TLD2") .def(init<>()) .def_readwrite("currConf", &tld::TLD2::currConf) .def("selectObject", &tld::TLD2::selectObject_numpy) .def("processImage", &tld::TLD2::processImage_numpy) .def("getCurrBB", &tld::TLD2::getCurrBB) .def("set_width_and_height", &tld::TLD2::set_width_and_height) .add_property("detectorCascade", make_getter(&tld::TLD::detectorCascade, return_value_policy()), make_setter(&tld::TLD::detectorCascade, return_value_policy())) ; } ================================================ FILE: other_trackers/setup_opentld.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from setuptools import find_packages from distutils.core import Extension, setup import os import cv2 this_dir = os.path.dirname(os.path.realpath(__file__)) cv_folder = [l[l.find('/'):] for l in cv2.getBuildInformation().splitlines() if 'Install path' in l][0] opentld_python_root = this_dir opentld_cpp_root = this_dir + "/original_opentld/src" opentld_source_folders = [opentld_cpp_root + '/libopentld/tld', opentld_cpp_root + '/libopentld/mftracker', opentld_cpp_root + '/3rdparty/cvblobs'] opentld_sources = [] for sf in opentld_source_folders: opentld_sources += [os.path.join(sf, i) for i in os.listdir(sf) if i.endswith('cpp')] opentld_module = Extension('tld', include_dirs=["/usr/include", "/usr/local/include", "/usr/include/boost", "/usr/include/opencv", "/usr/local/include/opencv", cv_folder + "/include", cv_folder + "/include/opencv", cv_folder + "/include/opencv2"] + opentld_source_folders, libraries=['boost_python', 'opencv_core', 'opencv_imgproc', 'opencv_video'], library_dirs=['/usr/local/lib', '/usr/lib', cv_folder + '/lib'], runtime_library_dirs=[cv_folder + '/lib'], sources=[opentld_python_root + '/opentld_python.cpp'] + opentld_sources) setup( name='Other tracker bindings', author='Brain Corporation', author_email='piekniewski@braincorporation.com', url='https://github.com/braincorp/PVM', long_description='', version='dev', packages=find_packages(), include_package_data=True, install_requires=[], ext_modules=[opentld_module]) ================================================ FILE: other_trackers/setup_struck.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from setuptools import find_packages from distutils.core import Extension, setup import os import cv2 from Cython.Build import cythonize import numpy this_dir = os.path.dirname(os.path.realpath(__file__)) cv_folder = [l[l.find('/'):] for l in cv2.getBuildInformation().splitlines() if 'Install path' in l][0] # struck tracker struck_src_dir = this_dir + "/original_struck/src/" struck_src = ["Tracker.cpp", "Config.cpp", "Features.cpp", "HaarFeature.cpp", "HaarFeatures.cpp", "HistogramFeatures.cpp", "ImageRep.cpp", "LaRank.cpp", "MultiFeatures.cpp", "RawFeatures.cpp", "Sampler.cpp", "GraphUtils/GraphUtils.cpp"] struck_all_src = [struck_src_dir+src for src in struck_src] struck_module = Extension("struck_bindings", sources=["./struck.cpp", "./struck_bindings.pyx"] + struck_all_src, language="c++", include_dirs=['.', numpy.get_include(), '/usr/include/eigen3/', "/usr/include/"], libraries=['opencv_core', 'opencv_highgui'], extra_compile_args=["-O2", "-m64"]) struck_module = cythonize([struck_module])[0] setup( name='Other tracker bindings', author='Brain Corporation', author_email='piekniewski@braincorporation.com', url='https://github.com/braincorp/PVM', long_description='', version='dev', packages=find_packages(), include_package_data=True, install_requires=[], ext_modules=[struck_module]) ================================================ FILE: other_trackers/struck.cpp ================================================ /* # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== */ #include "struck.h" #include "original_struck/src/Config.h" #include "opencv2/opencv.hpp" Tracker* tracker; Config conf; int width; int height; double scaleW; double scaleH; void struck_init(unsigned char* frame_data, int nrows, int ncols, const BoundingBox& bbox){ std::string configPath = "config.txt"; conf = Config(configPath); tracker = new Tracker(conf); height = nrows; width = ncols; scaleW = (double)conf.frameWidth/ncols; scaleH = (double)conf.frameHeight/nrows; cv::Mat original_frame; original_frame = cv::Mat(height, width, CV_8UC3, frame_data); cv::Mat frame; cv::resize(original_frame, frame, cv::Size(conf.frameWidth, conf.frameHeight)); FloatRect init_bb(bbox.xmin*scaleW, bbox.ymin*scaleH, bbox.width*scaleW, bbox.height*scaleH); tracker->Initialise(frame, init_bb); } void struck_track(unsigned char* frame_data){ if(!tracker->IsInitialised()){ std::cout << "Tracker is not initialized!!!" << std::endl; return; } cv::Mat original_frame; original_frame = cv::Mat(height, width, CV_8UC3, frame_data); cv::Mat frame; cv::resize(original_frame, frame, cv::Size(conf.frameWidth, conf.frameHeight)); tracker->Track(frame); // tracker->Debug(); } BoundingBox struck_get_bbox(){ FloatRect struck_bbox = tracker->GetBB(); BoundingBox bbox; bbox.xmin = static_cast(struck_bbox.XMin()/scaleW); bbox.ymin = static_cast(struck_bbox.YMin()/scaleH); bbox.width = static_cast(struck_bbox.Width()/scaleW); bbox.height = static_cast(struck_bbox.Height()/scaleH); return bbox; } ================================================ FILE: other_trackers/struck.h ================================================ /* # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== */ #ifndef STRUCK_H #define STRUCK_H #include "original_struck/src/Tracker.h" struct BoundingBox{ int width; /* width of the bounding box */ int height; /* height of the bounding box */ int xmin; /* x-coordinate of bottom left corner */ int ymin; /* y-coordinate of bottom left corner */ }; void struck_init(unsigned char* frame_data, int nrows, int ncols, const BoundingBox& bbox); void struck_track(unsigned char* frame_data); BoundingBox struck_get_bbox(); #endif //STRUCK_H ================================================ FILE: other_trackers/struck_bindings.pyx ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import numpy as np cimport numpy as np np.import_array() cdef extern from "struck.h": cdef void struck_init(unsigned char* frame_data, int nrows, int ncols, const BoundingBox& bbox); cdef void struck_track(unsigned char* frame_data); cdef BoundingBox struck_get_bbox(); ctypedef struct BoundingBox: int width; int height; int xmin; int ymin; def STRUCK_init(np.ndarray[unsigned char, ndim=3, mode='c'] frame not None, bbox): nrows = frame.shape[0] ncols = frame.shape[1] cdef BoundingBox struck_bbox; struck_bbox.xmin = bbox[0] struck_bbox.ymin = bbox[1] struck_bbox.width = bbox[2] struck_bbox.height = bbox[3] struck_init(&frame[0, 0, 0], nrows, ncols, struck_bbox); def STRUCK_track(np.ndarray[unsigned char, ndim=3, mode='c'] frame not None): struck_track(&frame[0, 0, 0]) def STRUCK_get_bbox(): cdef BoundingBox bbox; bbox = struck_get_bbox() return bbox ================================================ FILE: other_trackers/struck_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from PVM_tools.abstract_tracker import GenericVisionTracker import other_trackers.struck_bindings as struck import cv2 from PVM_tools.bounding_region import BoundingRegion struck_config = """ # quiet mode disables all visual output (for experiments). quietMode = 0 # debug mode enables additional drawing and visualization. debugMode = 1 # base path for video sequences. sequenceBasePath = sequences # path for output results file. # comment this out to disable output. #resultsPath = log.txt # video sequence to run the tracker on. # comment this out to use webcam. sequenceName = girl # frame size for use during tracking. # the input image will be scaled to this size. frameWidth = 320 frameHeight = 240 # seed for random number generator. seed = 0 # tracker search radius in pixels. searchRadius = 30 # SVM regularization parameter. svmC = 100.0 # SVM budget size (0 = no budget). svmBudgetSize = 100 # image features to use. # format is: feature kernel [kernel-params] # where: # feature = haar/raw/histogram # kernel = gaussian/linear/intersection/chi2 # for kernel=gaussian, kernel-params is sigma # multiple features can be specified and will be combined feature = haar gaussian 0.2 #feature = raw gaussian 0.1 #feature = histogram intersection """ class StruckTracker(GenericVisionTracker): """ TODO """ def __init__(self): """ Initialize the tracker """ self.name = 'struck_tracker' f = open("config.txt", "w") f.write(struck_config) f.close() self.reset() def reset(self): """ Reset the tracker :return: """ self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ self.bbox=bounding_region bounding_box = bounding_region.get_box_pixels() struck.STRUCK_init(im, bounding_box) def _track(self, im): """ Track on given image, return a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again self._primed = False struck.STRUCK_track(im) struck_bbox = struck.STRUCK_get_bbox() self.bounding_box = [struck_bbox["xmin"], struck_bbox["ymin"], struck_bbox["width"], struck_bbox["height"]] if self.bounding_box == (): self.bbox = BoundingRegion() else: self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box) return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.bbox.get_mask() ================================================ FILE: other_trackers/test_struck_bindings.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import time import cv2 if __name__ == "__main__": import other_trackers.struck_bindings as struck cam = cv2.VideoCapture(0) for i in range(40): ret, frame = cam.read() time.sleep(0.1) height = frame.shape[0] width = frame.shape[1] bbox_width = 80 bbox_height = 80 bbox = [(width-bbox_width)/2, (height-bbox_height)/2, bbox_width, bbox_height] struck.STRUCK_init(frame, bbox) while (True): ret, frame = cam.read() struck.STRUCK_track(frame) bbox = struck.STRUCK_get_bbox() cv2.rectangle(frame, (bbox["xmin"], bbox["ymin"]), (bbox["xmin"]+bbox["width"], bbox["ymin"]+bbox["height"]), (0, 1, 0)) cv2.imshow('original', frame) if cv2.waitKey(10) & 0xFF == ord('q'): break cam.release() cv2.destroyAllWindows() ================================================ FILE: other_trackers/test_tld_basic.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import pytest import logging import cv2 import numpy as np def test_tld(): import other_trackers.tld as tld tracker = tld.TLD2() initbb = [50, 50, 10, 10] img = np.random.randint(0, high=255, size=(100, 100, 3)).astype('uint8') height, width = img.shape[:2] tracker.set_width_and_height((width, height)) img_grey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img_cvmat = cv2.cv.fromarray(img_grey) tracker.selectObject(img_cvmat, tuple(initbb)) img = np.random.randint(0, high=255, size=(100, 100, 3)).astype('uint8') img_cvmat = cv2.cv.fromarray(img) tracker.processImage(img_cvmat) assert(tracker.getCurrBB() is not None) assert(tracker.currConf >= 0) if __name__ == '__main__': logging.basicConfig(level=logging.INFO) pytest.main('-s %s' % __file__) ================================================ FILE: other_trackers/tld_vision_tracker.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from PVM_tools.abstract_tracker import GenericVisionTracker import other_trackers.tld as tld import cv2 from PVM_tools.bounding_region import BoundingRegion class TLDVisionTracker(GenericVisionTracker): """ This class exposes the open tld tracker implemented in C++ by http://www.gnebehay.com/tld/. Originally OpenTLD that was originally published in MATLAB by Zdenek Kalal. """ def __init__(self): """ Initialize the tracker """ self.name = 'tld_tracker' self.reset() def reset(self): """ Reset the tracker :return: """ self.tld = None self._primed = False self.bbox = None def _prime(self, im, bounding_region): """ prime tracker on image and bounding box :param im: input image (3 - channel numpy array) :type im: numpy.ndarray :param bounding_region: initial bounding region of the tracked object :type bounding_region: PVM_tools.bounding_region.BoundingRegion """ self.bbox=bounding_region bounding_box = bounding_region.get_box_pixels() if not self._primed: self.tld = tld.TLD2() self._primed = True self.height, self.width = im.shape[:2] self.tld.set_width_and_height((self.width, self.height)) im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY) img_cvmat = cv2.cv.fromarray(im) if bounding_box[0] + bounding_box[2] > self.width: bounding_box[2] = self.width - bounding_box[0] if bounding_box[1] + bounding_box[3] > self.height: bounding_box[3] = self.height - bounding_box[1] self.tld.selectObject(img_cvmat, tuple([int(bounding_box[0]), int(bounding_box[1]), int(bounding_box[2]), int(bounding_box[3])])) def _track(self, im): """ Track on given image, return a bounding box :param im: image (3 - channel numpy array) :type im: numpy.ndarray :return: bounding box of the tracker object :rtype: PVM_tools.bounding_region.BoundingRegion """ # this is required so that tracker is re-initialized if it is primed again self._primed = False img_cvmat = cv2.cv.fromarray(im) self.tld.processImage(img_cvmat) self.bounding_box = self.tld.getCurrBB() self.confidence = self.tld.currConf if self.bounding_box == (): self.bbox = BoundingRegion() else: self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box, confidence=self.confidence) return self.bbox.copy() def get_heatmap(self, heatmap_name=None): return self.bbox.get_mask() ================================================ FILE: pytest.ini ================================================ # Configuration of py.test [pytest] # Run each test in an isolated environment # Ignore specialised tests # Our test plugins # Verbose addopts=-v --basetemp=.pytest_tmp --durations=10 # Be more specific about what tests to find python_files=test_*.py python_classes=Test python_functions=test_ norecursedirs= venv sphinx_doc # PEP-8 The following are ignored: # E251 unexpected spaces around keyword / parameter equals # E225 missing whitespace around operator # E226 missing whitespace around arithmetic operator # W291 trailing whitespace # W293 blank line contains whitespace # E501 line too long (82 > 79 characters) pep8ignore=* E251 \ * E225 \ * E226 \ * W291 \ * W293 \ * E501 \ generate_modules.py ALL \ other_trackers/original_cmt/* ALL \ other_trackers/original_opentld/* ALL\ other_trackers/original_struck/* ALL\ docs/* ALL ================================================ FILE: requirements.txt ================================================ # pip requirements file # example to include bc repos. Don't forget egg name (the same as repo name) # Test functionality pep8==1.4.6 pytest==2.3.5 pytest-cache==0.9 pytest-capturelog==0.7 pytest-cov==1.6 pytest-pep8==1.0.4 sphinx==1.3.1 mock==0.8.0 numpy==1.9.1 posix_ipc==0.9.9 cython==0.22.1 fusepy==2.0.2 ipython==3.1.0 line_profiler==1.0 matplotlib==1.5 ================================================ FILE: setup.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== from setuptools import setup, find_packages from distutils.core import Extension import os from Cython.Build import cythonize from Cython.Compiler.Main import default_options import numpy import platform def ver_to_float(ver): nums = ver.split('.') result = 0.0 base = 1.0 for num in nums: result += float(num)*base base *= 0.001 return result default_options['emit_linenums'] = True gcc_ver = os.popen("gcc -dumpversion").read() SSE_FLAGS = [] if platform.machine() == 'x86_64': SSE_FLAGS.append("-m64") SSE_FLAGS.append("-msse") SSE_FLAGS.append("-msse2") SSE_FLAGS.append("-msse3") SSE_FLAGS.append("-mfpmath=sse") Extensions = [Extension("PVM_framework/fast_routines", sources=["PVM_framework/fast_routines.pyx", "PVM_framework/Accelerated.cpp"], language="c++", include_path=['.', numpy.get_include()], extra_compile_args=["-O3"]+SSE_FLAGS)] if (ver_to_float(gcc_ver) < ver_to_float("4.7")): print "gcc version > 4.7 is nescessary for some extensions" else: Extensions.append(Extension("SyncUtils", sources=["PVM_framework/SyncUtils.pyx", "PVM_framework/Sync.cpp"], language="c++", include_path=['.', numpy.get_include()], extra_compile_args=["-O3"]+SSE_FLAGS)) if platform.machine() == 'x86_64' or platform.machine() == 'i386': Extensions.append(Extension("LowLevelCPUControlx86", sources=["PVM_framework/LowLevelCPUControlx86.pyx"], language="c++", include_path=['.', numpy.get_include()], extra_compile_args=["-O3"] + SSE_FLAGS)) setup( ext_package="PVM_framework", ext_modules=cythonize(Extensions), include_dirs=[numpy.get_include()], name='PVM_framework', author='Brain Corporation', author_email='piekniewski@braincorporation.com', url='https://github.com/braincorp/PVM', long_description='', version='1.0', packages=find_packages(), include_package_data=True, install_requires=[]) ================================================ FILE: test ================================================ #!/bin/bash # Check for python27 (so this works on centos) if type python27 > /dev/null 2>/dev/null ; then PYTHONEXEC=python27 else PYTHONEXEC=python fi $PYTHONEXEC -m pytest --junitxml=testresults.xml ================================================ FILE: tracker_tools/__init__.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== ================================================ FILE: tracker_tools/export_to_zip.py ================================================ """ Exports a movie in pickled labeled movie format to a zip archive containing the images and text description of the bounding boxes for each target. This format is often used in academic circles for sharing data. Usage: :: python export_to_zip.py -i my_file.pkl -o myfile.zip -c low_res """ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== import zipfile from PVM_tools import labeled_movie import PVM_framework.PVM_Storage as PVM_Storage import PVM_framework.PVM_datasets as PVM_datasets import argparse import cv2 import time import os def export_to_zip(infile, outfile, channel, compression_level=80): """ Exports a movie in pickled labeled movie format to a zip archive containing the images and text description of the bounding boxes for each target. This format is often used in academic circles for sharing data. :param infile: name of the input pickle :param outfile: name of the output zip archive :param channel: channel to be processed :param compression_level: jpeg compression level :return: """ fc = labeled_movie.FrameCollection() fc.load_from_file(infile) myzip = zipfile.ZipFile(outfile, "w") folder = os.path.basename(outfile)[:-4] targets = fc.Frame(0).get_targets(channel=channel) poster_frame = fc.Frame(0).get_image(channel=channel) label_str = {} for target in targets: label_str[target] = "" for i in xrange(len(fc)): zipi = zipfile.ZipInfo() zipi.filename = folder+"/img/%04d.jpg" % i zipi.date_time = time.localtime()[:6] zipi.compress_type = zipfile.ZIP_DEFLATED zipi.external_attr = 0777 << 16L img = fc.Frame(i).get_image(channel=channel) for target in targets: label = fc.Frame(i).get_label(target=target) if label.empty: label_str[target] += "-1, -1, -1, -1\n" else: box = label.get_box_pixels() label_str[target] += "%d, %d, %d, %d\n" % box (ret, buf) = cv2.imencode(".jpg", img, (cv2.IMWRITE_JPEG_QUALITY, compression_level)) myzip.writestr(zipi, buf) for (i, target) in enumerate(targets): zipi = zipfile.ZipInfo() zipi.filename = folder+"/groundtruth_rect.%d.txt" % i zipi.date_time = time.localtime()[:6] zipi.compress_type = zipfile.ZIP_DEFLATED zipi.external_attr = 0777 << 16L myzip.writestr(zipi, label_str[target]) myzip.close() poster_frame = cv2.resize(poster_frame, dsize=(120, 120), interpolation=cv2.INTER_CUBIC) cv2.imwrite(outfile.split('.')[0]+".jpg", poster_frame) if __name__ == "__main__": desc = """ Exports a movie in pickled labeled movie format to a zip archive containing the images and text description of the bounding boxes for each target. This format is often used in academic circles for sharing data. Usage: python export_to_zip.py -i my_file.pkl -o myfile.zip -c low_res """ parser = argparse.ArgumentParser(description=desc) parser.add_argument("-c", "--channel", type=str, default="default", help="Channel") parser.add_argument("-i", "--input", type=str, default="example.pkl", help="Input labeled movie") parser.add_argument("-o", "--output", type=str, default="example.zip", help="Output zip archive") parser.add_argument("-s", "--set", type=str, default="", help="Dataset") parser.add_argument("-O", "--out_folder", type=str, default="", help="Folder to store multiple files") parser.add_argument("-p", "--print_html", help="Print tr/td html elements to paste into a website", action="store_true") args = parser.parse_args() if args.set == "": export_to_zip(infile=args.input, outfile=args.output, channel=args.channel) else: storage = PVM_Storage.Storage() dataset = PVM_datasets.PVMDataset(name="all") for (i, [f, c]) in enumerate(dataset.all): if not args.print_html: print "Processing file %s" % f local_path = storage.get(f) out_filename = os.path.splitext(os.path.basename(local_path))[0] out_filename_zip = out_filename+".zip" out_complete = os.path.expanduser(os.path.join(args.out_folder, out_filename_zip)) export_to_zip(infile=local_path, outfile=out_complete, channel=args.channel) if args.print_html: if i % 6 == 0: print "" out_filename_add = "PVM_set/" + out_filename print "

"+out_filename+"

" ================================================ FILE: tracker_tools/images_to_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ Simple tool to convert sets of images and text ground truth rectangle files into the new format (somewhat an opposite of the export_to_zip.py file. Example: :: python images_to_PVM_pickle.py -l Example/groundtruth_rect.txt -i Example/img/ -o Example.pkl Where -l is the path to the file with labels, -i is the directory with images, and -o is the output file. Additional argument -s may be given to skip certain number of frames (in case the first line in the label file does not correspond with the first image). Usage: :: usage: images_to_PVM_pickle.py [-h] [-l LABELS] [-i IMAGES] [-o OUTPUT] [-s SKIP] optional arguments: -h, --help show this help message and exit -l LABELS, --labels LABELS File containing the rectangles -i IMAGES, --images IMAGES Directory containing the images -o OUTPUT, --output OUTPUT Output file -s SKIP, --skip SKIP Skip frames (labels only begin at some point) """ import argparse from PVM_tools.bounding_region import BoundingRegion from PVM_tools.labeled_movie import LabeledMovieFrame, FrameCollection import os import cv2 import numpy as np class ImagesToLabeledMovie(object): def __init__(self, label_file=None, image_dir=None, output_file=None, skip=None): if label_file is None or image_dir is None or output_file is None: raise Exception("Nescessary arguments are missing") self.label_file=open(label_file, "r") self.image_dir = image_dir self.output_file = output_file self.frame_collection = FrameCollection() self.file_index = 1 if skip is not None: self.skip = int(skip) else: self.skip = 0 def get_next_image_file(self): p = self.image_dir if not p.endswith("/"): p += "/" p += ("%04d.jpg" % self.file_index) self.file_index += 1 if os.path.isfile(p): return p else: return None def run(self): i = 0 while True: p = self.get_next_image_file() if p is None: break img = cv2.imread(p) if i >= self.skip: rect = self.label_file.readline() try: box = map(lambda x: int(x), rect.split("\t")) B = BoundingRegion(image_shape=img.shape, box=np.array(box)) except: try: box = map(lambda x: int(x), rect.split(",")) B = BoundingRegion(image_shape=img.shape, box=np.array(box)) except: print "No more bounding boxes!" B = BoundingRegion() else: B = BoundingRegion() F = LabeledMovieFrame(internal_storage_method='jpg', compression_level=90) F.set_image(img) F.set_label(B) B.draw_box(img) cv2.imshow("image", img) cv2.moveWindow("image", 50, 50) cv2.waitKey(1) i += 1 self.frame_collection.append(F) self.frame_collection.write_to_file(self.output_file) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-l", "--labels", type=str, help="File containing the rectangles") parser.add_argument("-i", "--images", type=str, help="Directory containing the images") parser.add_argument("-o", "--output", type=str, help="Output file") parser.add_argument("-s", "--skip", type=str, help="Skip frames (labels only begin at some point)") args = parser.parse_args() app = ImagesToLabeledMovie(label_file=args.labels, image_dir=args.images, output_file=args.output, skip=args.skip) app.run() ================================================ FILE: tracker_tools/label_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ Labeling movies ######## The label_PVM_pickle.py script is an easy to use tool for attaching labels to movies in labeled movie format. The process is easy and is executed in the following steps. 1. First identify a movie file in a regular format (e.g. mov, avi). Next use a conversion tool to convert it into a pickle. E.g. if the file was called example.mov one could run: :: python movie_to_PVM_pickle.py example.mov -o example.pkl This tool allows for additional processing, e.g. rotations, flipping, scaling. Refer to the documentation. 2. Once the pickle has been created, it can be tagged (labeled). Run: :: python label_PVM_pickle.py example.pkl A gui should show up, presenting the first frame of the movie. Mouse cursor should be surrounded by a box. By left clicking in a given position, the target bounding box is saved and next frame is displayed. The size of the current box can be defined by clicking in the left top corner of the target object with the middle button, and extending the box to cover the entire object. Additionally the box can be scaled up or down using 'a' and 'z' keys. Use right click to indicate that the target object is not present in the scene, whenever this is the case. Be careful on frames in which target disappears and reappears. If some parts of the movie were not labeled correctly, you can move back using cursor keys or scroll bar at the top of the window. After the process is complete, press 'q' key and 'y' to save the changes in the file. example.pkl is now modified with default target label on the default channel. 3. Once one channel has been labeled you may want to create e.g. a downscaled version of that channel but inherit and scale all the labels. scale_PVM_pickle.py script can be useful for that. Simply run: :: python scale_PVM_pickle.py example.pkl -i default -o med_res -d 0.5 -s Refer to the scale_PVM_pickle.py documentation for details. The options in the example above are: * -i input channel - the name of the channel being processed * -o output channel - the name of the new channel that will be created * -d 0.5 - scale content by a factor of 0.5 * -s - set the new channel as default. If the input channel was default and had no other name it will be renamed to channel01. 4. The file is ready and can be used in tracker benchmark. Refer to TSartifacts documentation for information on uploading the file to S3 bucket. 5. If you want to quickly view the content of the file and labels, use play_PVM_pickle.py script simply typing a command: :: python play_PVM_pickle.py example.pkl -b -b option will display the boxes. Multiple targets ######## The labeled movie format allows to store multiple target labels per channel. This allows to track multiple objects. In order to label multiple objects follow the steps above, after step 2 execute: :: python label_PVM_pickle.py example.pkl -t target1 with the -t option to select the new target. The labeling tool will open without anything labeled. Proceed labeling the additional objects of interest. Once done, remember to save your work. Files with multiple labels can also by scaled by the scale_PVM_pickle.py tool. It will scale all the labels obtained from the source channel. .. note:: It's a good habit to carefully label the high resolution version of the video and then use such high quality data, to create lower resolution channels. That way the labels in the lower resolution channels will be very precise. If you don't want to modify the source file but instead write to another file, use the -o option when calling the tool: :: python label_PVM_pickle.py example.pkl -t target1 -o new_file.pkl .. note:: The labels in the labeled movies can always be modified and improved. Try not to generate too many files with the same content but different versions of labels. Instead rather keep everything in one place (file) and make sure the labels are of highest possible quality. Script usage message ######## :: Usage: python label_movie.py input_file.pkl [-o output_file] [-c channel] Keys: left, right cursor - move to previous/next frame t - toggle a tracker [space] - validate the tracker bounding box and move to the next frame w - write movie to the output file (same as input if not given) a - scale current box up by 10% in both dimensions z - scale current box down by 10% in both dimensions s - scale current box up by 10% in the vertical direction x - scale current box down by 10% in the vertical direction q - quit """ import argparse from PVM_tools.labeled_movie import FrameCollection from PVM_tools.bounding_region import BoundingRegion import cv2 import threading import logging import numpy as np import time class LabelingApp(object): def __init__(self, filename, output=None, channel="default", target="default"): self.input_filename = filename if output is None: self.output_file = self.input_filename else: self.output_file = output self.channel = channel self.target = target self.reset() def reset(self, reload=False): cv2.destroyAllWindows() fc = FrameCollection() if reload: fc.load_from_file(filename=self.output_file) else: fc.load_from_file(filename=self.input_filename) self.movie = fc self.tracker = None self.current_frame_index = 0 self.right_button_pressed = False self.left_button_pressed = False self._anchor = None self.x_size = 30 self.y_size = 30 self._tracked_bounds = None self._stored_bounds = None self._current_bounds = BoundingRegion() # Make sure self.timer is set self.schedule_callback() # Initialization of the Graphic User Interface self.ready_to_refresh = threading.Lock() self.timer_lock = threading.Lock() self.win_name = 'Labeling video GUI' self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel) self.display_image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel) self.image_shape = self.image.shape self.create_windows() self.set_target_absent() self.image_buffer = {} self._last_update = time.time() self.refresh_timer = threading.Timer(0.05, self.refresh) self.refresh_timer.start() self.needs_refresh = True self.trim_end = len(self.movie)-1 self.trim_start = 0 def fill_up_the_buffer(self): r0 = max(self.current_frame_index-5, 0) r1 = min(self.current_frame_index+5, len(self.movie)) for i in xrange(r0, r1, 1): if i not in self.image_buffer.keys(): self.image_buffer[i] = self.movie.Frame(i).get_image(channel=self.channel) for i in self.image_buffer.keys(): if i < r0 or i > r1: del self.image_buffer[i] def create_windows(self): ''' Create playback progress bar (allow user to move along the movie) ''' cv2.namedWindow(self.win_name) cv2.moveWindow(self.win_name, 150, 150) cv2.namedWindow("completeness") cv2.moveWindow("completeness", 150, 50) cv2.setMouseCallback(self.win_name, self.trackbar_window_onMouse) cv2.createTrackbar("Frame", self.win_name, 0, len(self.movie) - 1, self.jump_to_frame_callback) cv2.createTrackbar("Trim start", self.win_name, 0, len(self.movie) - 1, self.set_trim_start) cv2.createTrackbar("Trim end", self.win_name, len(self.movie) - 1, len(self.movie) - 1, self.set_trim_end) im_height = 30 # 30 pixels high completeness bar im_width = len(self.movie) self.completeness_image = np.zeros((im_height, im_width, 3)) current_frame_index = int(im_width * float(self.current_frame_index) / len(self.movie)) self.completeness_image[:, current_frame_index, :] = 1. self.completeness = np.zeros(im_width, dtype=np.uint8) for i in xrange(len(self.movie)): bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target) if bounds is None or bounds.empty: if bounds is None: self.movie.Frame(i).set_label(channel=self.channel, target=self.target, label=BoundingRegion()) self.completeness[i] = 2 else: if bounds.is_keyframe(): self.completeness[i] = 1 else: self.completeness[i] = 0 def set_trim_start(self, frame_index): self.trim_start = frame_index + 0 def set_trim_end(self, frame_index): self.trim_end = frame_index + 0 def jump_to_frame_callback(self, frame_index): self.current_frame_index = frame_index self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel) self.needs_refresh = True def update_completeness_window(self): if self.ready_to_refresh.acquire(False): self.completeness_image *= 0 self.completeness_image[:, self.current_frame_index, :] = 255 for i in xrange(len(self.movie)): self.completeness_image[:, i, self.completeness[i]] = 255 cv2.putText(self.completeness_image, '%i' % self.current_frame_index, (10, 10), cv2.FONT_HERSHEY_PLAIN, 1.0, (255, 255, 255), lineType=cv2.CV_AA) cv2.imshow('completeness', self.completeness_image) self.ready_to_refresh.release() def update_trackbar_window(self): if self.ready_to_refresh.acquire(): image = self.image.copy() self._stored_bounds = self.movie.Frame(self.current_frame_index).get_label(channel=self.channel, target=self.target) # display rectangle over selected area self._current_bounds.draw_box(image, color=(255, 255, 255)) self._current_bounds.draw_box(image, color=(0, 0, 0), thickness=1) # display rectangle over stored area if self._stored_bounds is not None: self._stored_bounds.draw_box(image, color=(255, 0, 0)) if self._tracked_bounds is not None: self._tracked_bounds.draw_box(image, color=(0, 255, 0)) # reshape to display if too small min_width = 320 if image.shape[1] < min_width: resize_height = int(0.5 + image.shape[0] * min_width / float(image.shape[1])) image = cv2.resize(image, (min_width, resize_height), interpolation=cv2.INTER_NEAREST) self.display_image=image self.ready_to_refresh.release() def refresh(self): if self.timer_lock.acquire(): if self.needs_refresh: self.update_trackbar_window() self.update_completeness_window() self.needs_refresh = False self.refresh_timer = threading.Timer(0.05, self.refresh) self.refresh_timer.start() self.timer_lock.release() def trackbar_window_onMouse(self, event, x, y, flags, _): cv2.imshow(self.win_name, self.display_image) # Update state self.right_button_pressed = (flags & cv2.EVENT_FLAG_RBUTTON) != 0 and event != cv2.EVENT_RBUTTONUP self.left_button_pressed = (flags & cv2.EVENT_FLAG_LBUTTON) != 0 and event != cv2.EVENT_LBUTTONUP # Set bounding box if event == cv2.EVENT_MBUTTONDOWN: if self._anchor is None: # Setting the anchor self._anchor = (x, y) else: # Defining the box self.save_and_advance() elif self._anchor is not None and event != cv2.EVENT_MBUTTONUP: # We are creating new bounding box from anchor to current position self.x_size = max(5, int(abs(x - self._anchor[0])+0.5)) self.y_size = max(5, int(abs(y - self._anchor[1])+0.5)) center_x = (x + self._anchor[0]) / 2 center_y = (y + self._anchor[1]) / 2 self.set_bounding_box_with_center(center_x, center_y) elif event == cv2.EVENT_MBUTTONUP: self.x_size = max(5, int(abs(x - self._anchor[0])+0.5)) self.y_size = max(5, int(abs(y - self._anchor[1])+0.5)) center_x = (x + self._anchor[0]) / 2 center_y = (y + self._anchor[1]) / 2 self.set_bounding_box_with_center(center_x, center_y) keyframe = False if flags & cv2.EVENT_FLAG_CTRLKEY != 0: keyframe = True self.save_and_advance(keyframe=keyframe) else: self.set_bounding_box_with_center(x, y) # If this is a left/right button down event, # we can advance one frame immediately # and then start the timer if event == cv2.EVENT_RBUTTONDOWN: self.set_target_absent() self.save_and_advance() self.right_button_pressed=True if self.timer.finished: # Rearm the timer self.schedule_callback() elif event == cv2.EVENT_LBUTTONDOWN: self.set_bounding_box_with_center(x, y) keyframe = False if flags & cv2.EVENT_FLAG_CTRLKEY != 0: keyframe = True self.save_and_advance(keyframe=keyframe) self.left_button_pressed=True if self.timer.finished: # Rearm the timer self.schedule_callback() elif not (self.right_button_pressed or self.left_button_pressed): # No mouse button pressed, so cancel the timer if it's running self.timer.cancel() if time.time()-self._last_update > 0.1: self._last_update = time.time() self.needs_refresh = True def set_target_absent(self): self._current_bounds = BoundingRegion() def set_bounding_box_with_center(self, x, y): box = [np.clip(int(x+0.5)-self.x_size/2, 0, self.image_shape[1]), np.clip(int(y+0.5)-self.y_size/2, 0, self.image_shape[0]), self.x_size + min(int(x+0.5)-self.x_size/2, 0), self.y_size + min(int(y+0.5)-self.y_size/2, 0)] self._current_bounds = BoundingRegion(box=box) self.needs_refresh = True def schedule_callback(self): # 0.2 secs => 5 Hz self.timer = threading.Timer(0.1, self.timer_callback) self.timer.start() def timer_callback(self): if self.right_button_pressed or self.left_button_pressed: self.save_and_advance() self.schedule_callback() def advance_current_frame(self, increment=1): self.current_frame_index = min(len(self.movie)-1, max(0, self.current_frame_index + increment)) self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel) cv2.setTrackbarPos("Frame", self.win_name, self.current_frame_index) if self.tracker is not None: self._tracked_bounds = self.tracker.track(self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)) def save_and_advance(self, keyframe=False): self._anchor = None self.movie.Frame(self.current_frame_index).set_label(self._current_bounds.copy(), channel=self.channel, target=self.target) if self._current_bounds.empty: self.completeness[self.current_frame_index] = 2 else: if self._current_bounds.is_keyframe(): self.completeness[self.current_frame_index] = 1 else: self.completeness[self.current_frame_index] = 0 if keyframe: self.make_keyframe() self.advance_current_frame(15) else: self.advance_current_frame(1) def toggle_tracker(self): if self.tracker is None: self.tracker = CMTVisionTracker() self.tracker.prime(self.movie.Frame(self.current_frame_index).get_image(channel=self.channel), self._current_bounds) logging.warning("Tracker is enabled") else: self.tracker = None self._tracked_bounds = None logging.warning("Tracker is disabled") def advance_tracker(self): if self._tracked_bounds is not None: self._current_bounds=self._tracked_bounds self.save_and_advance() def export_movie(self): trim_frames_front = self.trim_start trim_frames_end = (len(self.movie) -1) - self.trim_end if trim_frames_end > 0: for i in range(trim_frames_end): self.movie.delete(-1) if trim_frames_front>0: for i in range(trim_frames_front): self.movie.delete(0) logging.warning("Exporting result in file " + self.output_file) self.movie.write_to_file(self.output_file) logging.warning("Exporting completed !") def interpolate(self, start, end): print "Interpolating %d %d" % (start, end) label0 = self.movie.Frame(start).get_label(channel=self.channel, target=self.target) label1 = self.movie.Frame(end).get_label(channel=self.channel, target=self.target) if (label0.empty) or (label1.empty): return box0 = label0.get_box_pixels() box1 = label1.get_box_pixels() for i in xrange(start+1, end, 1): alpha = (end-i)*1.0/(end-start) box2 = alpha*np.array(box0)+(1-alpha)*np.array(box1) box2 = map(lambda x: int(x), box2) self.movie.Frame(i).set_label(BoundingRegion(box=box2, image_shape=self.movie.Frame(i).get_image(channel=self.channel).shape), channel=self.channel, target=self.target) self.completeness[i] = 0 def make_keyframe(self): self.completeness[self.current_frame_index] = 1 self.movie.Frame(self.current_frame_index).get_label(channel=self.channel, target=self.target).set_keyframe(True) # find previous keyframe next_keyframe = -1 for i in xrange(self.current_frame_index+1, len(self.movie), 1): bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target) if bounds.is_keyframe(): next_keyframe = i break if next_keyframe >= 0: print next_keyframe self.interpolate(self.current_frame_index, next_keyframe) # find next keyframe prev_keyframe = -1 for i in xrange(self.current_frame_index-1, -1, -1): bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target) if bounds.is_keyframe(): prev_keyframe = i break if prev_keyframe >= 0: print prev_keyframe self.interpolate(prev_keyframe, self.current_frame_index) def run(self): while True: key = cv2.waitKey(0) cv2.imshow(self.win_name, self.display_image) if key == 65361: # left arrow # Go to previous frame self.advance_current_frame(-1) elif key == 65362: self.advance_current_frame(-10) elif key == 65363: # right arrow # Go to next frame self.advance_current_frame() elif key == 65364: self.advance_current_frame(10) elif key == ord('w'): # Write self.export_movie() if self.timer_lock.acquire(): if not self.refresh_timer.finished: self.refresh_timer.cancel() self.timer_lock.release() self.timer.cancel() self.reset() elif key == ord('t'): # Tracker self.toggle_tracker() elif key == ord(' '): # Advance tracker self.advance_tracker() self.needs_refresh = True elif key == ord('a'): self.x_size = int(self.x_size*1.1) self.y_size = int(self.y_size*1.1) self.needs_refresh = True elif key == ord('z'): self.x_size = int(self.x_size*1/1.1) self.y_size = int(self.y_size*1/1.1) self.needs_refresh = True elif key == ord('s'): self.y_size = int(self.y_size*1.1) self.needs_refresh = True elif key == ord('k'): self.make_keyframe() elif key == ord('x'): self.y_size = int(self.y_size*1/1.1) self.needs_refresh = True elif key == ord('q'): # Export while key != -1: for k in range(10): key = cv2.waitKey(10) print "Quiting! Do you want to save your work? [Y/N]" while True: key = cv2.waitKey(0) if key == ord('Y') or key == ord('y'): self.export_movie() break if key == ord('N') or key == ord('n'): break break logging.warning('Exiting') self.refresh_timer.cancel() if __name__ == "__main__": doc = """ Usage: python label_movie.py input_file.pkl [-o output_file] [-c channel] Keys: left, right cursor - move to previous/next frame t - toggle a tracker [space] - validate the tracker bounding box and move to the next frame w - write movie to the output file (same as input if not given) a - scale current box up by 10% in both dimensions z - scale current box down by 10% in both dimensions s - scale current box up by 10% in the vertical direction x - scale current box down by 10% in the vertical direction q - quit """ parser = argparse.ArgumentParser(description=doc) parser.add_argument('input_file', default='demo.pkl', nargs=1, help='Input file') parser.add_argument("-o", "--output", type=str, help="Output file (otherwise will save changes in the input file)") parser.add_argument("-c", "--channel", type=str, default="default", help="Channel") parser.add_argument("-t", "--target", type=str, default="default", help="Name of the labeled target") args = parser.parse_args() if not args.input_file: parser.print_help() else: print doc app = LabelingApp(filename=args.input_file[0], output=args.output, channel=args.channel, target=args.target) app.run() ================================================ FILE: tracker_tools/movie_to_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ Convert a movie in traditional format (e.g. avi or mov) to a labeled movie archive that can later be tagged with target labels. This script allows for some pre-processing, like rotating or flipping, changing resolution. Example: :: python movie_to_pickle -o mymovie.pkl -r 180 -f jpg -q 85 -d 720x460 IMG_1098.MOV The above will convert a file IMG_1098.MOV, rotate 180 degrees (videos e.g. from iphone will often be upside down) rescale to 720x460 and save in a pickle as jpg with quality 85. Usage :: usage: movie_to_PVM_pickle.py [-h] [-o OUTPUT] [-c CHANNEL] [-d DSIZE] [-r ROTATE] [-f FORMAT] [-q QUALITY] [-fv] [-fh] input_file positional arguments: input_file Input movie file optional arguments: -h, --help show this help message and exit -o OUTPUT, --output OUTPUT Output file -c CHANNEL, --channel CHANNEL Channel -d DSIZE, --dsize DSIZE Destination size e.g. 500x300 -r ROTATE, --rotate ROTATE Rotate by 90, 180 or 270 degrees -f FORMAT, --format FORMAT Internal storage format of the pickle -q QUALITY, --quality QUALITY Quality 0-100, larger - better image but bigger file -fv, --flip_vertical Flip vertical -fh, --flip_horizontal Flip horizontal """ import cv2 import PVM_tools.labeled_movie as lm import argparse import numpy as np def convert_to_pickle(infilename, outfilename, dsize, channel, rotate, format, quality, flip_vert, flip_hor): cam = cv2.VideoCapture(infilename) fc = lm.FrameCollection() cv2.namedWindow("Image") cv2.moveWindow("Image", 50, 50) while True: (ret, im) = cam.read() if not ret: break if dsize is not None: osize = tuple(map(lambda x: int(x), dsize.split("x"))) im=cv2.resize(im, dsize=osize, interpolation=cv2.INTER_CUBIC) if rotate is not None and rotate in ["90", "180", "270"]: im = np.rot90(im) if rotate in ["180", "270"]: im = np.rot90(im) if rotate in ["270"]: im = np.rot90(im) if flip_vert: im = cv2.flip(im, 0) if flip_hor: im = cv2.flip(im, 1) cv2.imshow("Image", im) key = cv2.waitKey(1) if key == 27: quit() f = lm.LabeledMovieFrame(internal_storage_method=format, compression_level=int(quality)) f.create_channel(channel=channel) f.set_image(im, channel=channel) f.set_default_channel(channel=channel) fc.append(f) fc.write_to_file(outfilename) if __name__ == "__main__": desc = """ Convert a movie in traditional format (e.g. avi or mov) to a labeled movie archive that can later be tagged with target labels. This script allows for some pre-processing, like rotating or flipping, changing resolution. Example: python movie_to_pickle -o mymovie.pkl -r 180 -f jpg -q 85 -d 720x460 IMG_1098.MOV The above will convert a file IMG_1098.MOV, rotate 180 degrees (videos e.g. from iphone will often be upside down) rescale to 720x460 and save in a pickle as jpg with quality 85. """ parser = argparse.ArgumentParser(description=desc) parser.add_argument('input_file', default='some_movie_file', nargs=1, help='Input movie file') parser.add_argument("-o", "--output", type=str, help="Output file") parser.add_argument("-c", "--channel", type=str, default="default", help="Channel ") parser.add_argument("-d", "--dsize", type=str, help="Destination size e.g. 500x300 ") parser.add_argument("-r", "--rotate", type=str, help="Rotate by 90, 180 or 270 degrees ") parser.add_argument("-f", "--format", type=str, default="jpg", help="Internal storage format of the pickle") parser.add_argument("-q", "--quality", type=str, default="80", help="Quality 0-100, larger - better image but bigger file") parser.add_argument("-fv", "--flip_vertical", help="Flip vertical", action="store_true") parser.add_argument("-fh", "--flip_horizontal", help="Flip horizontal", action="store_true") args = parser.parse_args() if not args.input_file: parser.print_help() else: convert_to_pickle(infilename=args.input_file[0], outfilename=args.output, channel=args.channel, dsize=args.dsize, rotate=args.rotate, format=args.format, quality=args.quality, flip_vert=args.flip_vertical, flip_hor=args.flip_horizontal) ================================================ FILE: tracker_tools/play_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ This tool is used to play a pickled labeled frame collection Example: :: python play_PVM_pickle.py some_pickle.pkl -b -c some_channel The -b option will display the bounding boxes of all the labeled targets, -c selects the channel to play. """ import argparse from PVM_tools.labeled_movie import FrameCollection import cv2 import numpy as np import os if __name__ == "__main__": desc = """ This tool is used to play a pickled labeled frame collection """ parser = argparse.ArgumentParser(description=desc) parser.add_argument('input_file', nargs="*", default='demo.pkl', help='Input file') parser.add_argument("-b", "--box", help="Draw the bounding box", action="store_true") parser.add_argument("-c", "--channel", type=str, default="default", help="Channel ") parser.add_argument("-o", "--output", type=str, default="", help="Output file") parser.add_argument("-k", "--crop", action="store_true", help="Crop to bounding box, black frame is target absent") parser.add_argument("-s", "--size", type=str, default="200x200", help="Size of the crop, default 200x200") parser.add_argument("-t", "--target", type=str, default="default", help="Crop to bounding box, black frame is target absent") args = parser.parse_args() if not args.input_file: parser.print_help() else: cv2.namedWindow("Player") cv2.moveWindow("Player", 50, 50) box_shape = tuple(map(lambda x: int(x), args.size.split("x"))) _video = None fc = FrameCollection() for i in range(len(args.input_file)): fc.load_from_file(filename=args.input_file[i]) fc.set_active_channel(args.channel) for i in xrange(len(fc)): img = fc.Frame(i).get_image(channel=args.channel) if args.box: for target in fc.Frame(i).get_targets(): br = fc.Frame(i).get_label(channel=args.channel, target=target) br.draw_box(img, thickness=1, annotation=target) if args.crop: br = fc.Frame(i).get_label(channel=args.channel, target=args.target) if not br.empty: box = br.get_box_pixels() crop = img[box[1]:box[1]+box[3], box[0]:box[0]+box[2]] crop_res = cv2.resize(crop, dsize=box_shape, interpolation=cv2.INTER_CUBIC) img = crop_res else: img = np.zeros((box_shape[0], box_shape[1], 3), dtype=np.uint8) elif args.size: img = cv2.resize(img, dsize=box_shape, interpolation=cv2.INTER_CUBIC) cv2.imshow("Player", img) if args.output != "": if _video is None: _video = cv2.VideoWriter() fps = 20 retval = _video.open(os.path.expanduser(args.output), cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), fps, (img.shape[1], img.shape[0])) assert(retval) _video.write(img) key = cv2.waitKey(int(1000/fc.fps)) & 0xFF if key == 27: break ================================================ FILE: tracker_tools/raw_to_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ Convert the data in raw format to a labeled movie. This script allows for some pre-processing, like rotating or flipping, changing resolution. Example: :: python raw_to_PVM_pickle.py -i ~/tracker_data/CLIF_2006_SAMPLE_SET/RAW/ -o tmp.pkl -s 2672x4008 -r 90 -d 0.25 -fv -fh -p 000001 This example will take the files in directory ~/tracker_data/CLIF_2006_SAMPLE_SET/RAW/ that begin with 000001, load them as numpy arrays with shape 2672x4008, rotate by 90 deg, scale by a factor of 0.25, flip vertically, flip horizontally and save to tmp.pkl. Usage :: usage: raw_to_PVM_pickle.py [-h] [-i IMAGES] [-o OUTPUT] [-c CHANNEL] [-s SHAPE] [-d DSIZE] [-r ROTATE] [-f FORMAT] [-q QUALITY] [-p PREFIX] [-fv] [-fh] optional arguments: -h, --help show this help message and exit -i IMAGES, --images IMAGES Directory containing the images -o OUTPUT, --output OUTPUT Output file -c CHANNEL, --channel CHANNEL Output file -s SHAPE, --shape SHAPE Shape XxY -d DSIZE, --dsize DSIZE Destination size e.g. 500x300 or scale factor e.g 0.5 -r ROTATE, --rotate ROTATE Rotate by 90, 180 or 270 degrees -f FORMAT, --format FORMAT Internal storage format of the pickle -q QUALITY, --quality QUALITY Quality 0-100, larger - better image but bigger file -p PREFIX, --prefix PREFIX File prefix -fv, --flip_vertical Flip vertical -fh, --flip_horizontal Flip horizontal """ import argparse from PVM_tools.labeled_movie import LabeledMovieFrame, FrameCollection import os import cv2 import numpy as np class ImagesToLabeledMovie(object): def __init__(self, image_dir=None, output_file=None, skip=None, channel="default", file_prefix="000000", dsize="1.0", flip_vert=None, flip_hor=None, rotate=None, shape=(100, 100), format="jpg", quality=90): if image_dir is None or output_file is None: raise Exception("Nescessary arguments are missing") self.image_dir = image_dir self.output_file = output_file self.frame_collection = FrameCollection() self.channel = channel self.file_index = 1 if skip is not None: self.skip = int(skip) else: self.skip = 0 self.files = sorted([f for f in os.listdir(image_dir) if os.path.isfile(os.path.join(image_dir, f)) and f.startswith(file_prefix)]) self.shape = tuple(map(lambda x: int(x), shape.split("x"))) self.rotate = rotate self.flip_vert = flip_vert self.flip_hor = flip_hor self.dsize = dsize self.format = format self.quality = quality def get_next_image_file(self): if self.file_index < len(self.files): self.file_index += 1 return os.path.join(self.image_dir, self.files[self.file_index-2]) else: return None def run(self): i = 0 cv2.namedWindow("image") cv2.moveWindow("image", 50, 50) while True: p = self.get_next_image_file() if p is None: break img = np.memmap(p, dtype=np.uint8, mode="r", shape=self.shape) F = LabeledMovieFrame(internal_storage_method=self.format, compression_level=int(self.quality)) if self.rotate is not None and self.rotate in ["90", "180", "270"]: img = np.rot90(img) if self.rotate in ["180", "270"]: img = np.rot90(img) if self.rotate in ["270"]: img = np.rot90(img) if self.flip_vert: img = cv2.flip(img, 0) if self.flip_hor: img = cv2.flip(img, 1) if self.dsize.count("x") == 1: osize = tuple(map(lambda x: int(x), self.dsize.split("x"))) else: factor = float(self.dsize) osize = int(img.shape[1]*factor), int(img.shape[0]*factor) img = cv2.resize(img, dsize=osize, interpolation=cv2.INTER_CUBIC) F.set_image(img) cv2.imshow("image", img) cv2.waitKey(40) i += 1 self.frame_collection.append(F) self.frame_collection.write_to_file(self.output_file) if __name__ == "__main__": desc = """ Example: python raw_to_PVM_pickle.py -i ~/tracker_data/CLIF_2006_SAMPLE_SET/RAW/ -o tmp.pkl -s 2672x4008 -r 90 -d 0.25 -fv -fh -p 000001 """ parser = argparse.ArgumentParser() parser.add_argument("-i", "--images", type=str, help="Directory containing the images") parser.add_argument("-o", "--output", type=str, help="Output file") parser.add_argument("-c", "--channel", type=str, default="default", help="Output file") parser.add_argument("-s", "--shape", type=str, default="400x600", help="Shape XxY") parser.add_argument("-d", "--dsize", type=str, default="1.0", help="Destination size e.g. 500x300 or scale factor e.g 0.5 ") parser.add_argument("-r", "--rotate", type=str, help="Rotate by 90, 180 or 270 degrees ") parser.add_argument("-f", "--format", type=str, default="jpg", help="Internal storage format of the pickle") parser.add_argument("-q", "--quality", type=str, default="80", help="Quality 0-100, larger - better image but bigger file") parser.add_argument("-p", "--prefix", type=str, default="000000", help="File prefix") parser.add_argument("-fv", "--flip_vertical", help="Flip vertical", action="store_true") parser.add_argument("-fh", "--flip_horizontal", help="Flip horizontal", action="store_true") args = parser.parse_args() app = ImagesToLabeledMovie(image_dir=args.images, output_file=args.output, channel=args.channel, dsize=args.dsize, rotate=args.rotate, format=args.format, quality=args.quality, flip_vert=args.flip_vertical, flip_hor=args.flip_horizontal, shape=args.shape, file_prefix=args.prefix) app.run() ================================================ FILE: tracker_tools/scale_PVM_pickle.py ================================================ # ================================================================================== # Copyright (c) 2016, Brain Corporation # # This software is released under Creative Commons # Attribution-NonCommercial-ShareAlike 3.0 (BY-NC-SA) license. # Full text available here in LICENSE.TXT file as well as: # https://creativecommons.org/licenses/by-nc-sa/3.0/us/legalcode # # In summary - you are free to: # # Share - copy and redistribute the material in any medium or format # Adapt - remix, transform, and build upon the material # # The licensor cannot revoke these freedoms as long as you follow the license terms. # # Under the following terms: # * Attribution - You must give appropriate credit, provide a link to the # license, and indicate if changes were made. You may do so # in any reasonable manner, but not in any way that suggests # the licensor endorses you or your use. # * NonCommercial - You may not use the material for commercial purposes. # * ShareAlike - If you remix, transform, or build upon the material, you # must distribute your contributions under the same license # as the original. # * No additional restrictions - You may not apply legal terms or technological # measures that legally restrict others from # doing anything the license permits. # ================================================================================== """ This tool is used to create new channels within a movie containing scaled versions of the original channel. Channels can either be down or up scaled. All the target labels existing for the original channel are scaled accordingly. This can be useful if you have a high definition labeled video but you want to use a downscaled version thereof for e.g. testing a tracking algorithm. Usage: :: python scale_PVM_pickle.py my_file.pkl -i high_res -o low_res -d 0.5 -s Arguments: :: positional arguments: input_file Input movie file optional arguments: -h, --help show this help message and exit -i INPUT_CHANNEL, --input_channel INPUT_CHANNEL Input channel -o OUTPUT_CHANNEL, --output_channel OUTPUT_CHANNEL Output channel -d DSIZE, --dsize DSIZE Destination size e.g. 500x300 or 0.5 for downscaling by two -f FORMAT, --format FORMAT Internal storage format of the pickle -q QUALITY, --quality QUALITY Quality 0-100, larger - better image but bigger file -s, --set_default Set the new channel to be default """ import argparse import cv2 import PVM_tools.labeled_movie as lm def scale_content(filename, dsize, source_channel, destination_channel, format, grayscale, quality, set_default): fc = lm.FrameCollection() fc.load_from_file(filename) cv2.namedWindow("Image") cv2.moveWindow("Image", 50, 50) for i in xrange(len(fc)): frame = fc.Frame(i) im = frame.get_image(channel=source_channel) shape = im.shape if dsize.count("x") == 1: osize = tuple(map(lambda x: int(x), dsize.split("x"))) else: factor = float(dsize) osize = int(im.shape[1]*factor), int(im.shape[0]*factor) im = cv2.resize(im, dsize=osize, interpolation=cv2.INTER_CUBIC) if grayscale: img = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) im[:, :, 0] = img im[:, :, 1] = img im[:, :, 2] = img frame.create_channel(channel=destination_channel) frame.set_image(im, channel=destination_channel, storage_method=format) disp_im = im.copy() for target in frame.get_targets(channel=source_channel): br = frame.get_label(channel=source_channel, target=target) br = br.copy() br.scale_to_new_image_shape(new_image_shape=im.shape, old_image_shape=shape) frame.set_label(br, channel=destination_channel, target=target) br.draw_box(disp_im, color=(255, 0, 0), annotation=target) if set_default: frame.set_default_channel(channel=destination_channel) cv2.imshow("Image", disp_im) key = cv2.waitKey(1) if key == 27: quit() fc.write_to_file(filename) if __name__ == "__main__": doc = """ This tool is used to create new channels within a movie containing scaled versions of the original channel. Channels can either be down or up scaled. All the target labels existing for the original channel are scaled accordingly. This can be useful if you have a high definition labeled video but you want to use a downscaled version thereof for e.g. testing a tracking algorithm. Usage: :: python scale_PVM_pickle.py my_file.pkl -i high_res -o low_res -d 0.5 -s """ parser = argparse.ArgumentParser(description=doc) parser.add_argument('input_file', default='pickle file', nargs=1, help='Input movie file') parser.add_argument("-i", "--input_channel", type=str, help="Input channel") parser.add_argument("-o", "--output_channel", type=str, default="default", help="Output channel") parser.add_argument("-d", "--dsize", type=str, default="1.0", help="Destination size e.g. 500x300 or 0.5 for downscaling by two") parser.add_argument("-f", "--format", type=str, default="jpg", help="Internal storage format of the pickle") parser.add_argument("-b", "--bw", help="Convert to grayscale", action="store_true") parser.add_argument("-q", "--quality", type=str, default="80", help="Quality 0-100, larger - better image but bigger file") parser.add_argument("-s", "--set_default", help="Set the new channel to be default", action="store_true") args = parser.parse_args() if not args.input_file: parser.print_help() else: scale_content(filename=args.input_file[0], dsize=args.dsize, source_channel=args.input_channel, destination_channel=args.output_channel, format=args.format, quality=args.quality, grayscale=args.bw, set_default=args.set_default) ================================================ FILE: tracker_tools/upgrade_cloud_lib.py ================================================ """ This small tool will go over all the pkl files stores in the ts amazon bucket, download each one of them, load and upgrade the movie version to the latest, save the file and upload back to amazon bucket. Use this when something changes about the data format. Usage: :: python upgrade_cloud_lib.py -u -f Options: * -f force download to make sure all the files are actually those stored in S3 * -u upload files, otherwise will run dry, that is upgrade files only locally When running first time after some major changes in the format I suggest to skip -u option for the first run, to make sure all the movies convert without issues. """ import PVM_tools.labeled_movie as lm import PVM_framework.PVM_datasets as PVM_datasets import PVM_framework.PVM_Storage as PVM_Storage import argparse import cv2 import os import logging if __name__ == "__main__": doc=""" This small tool will go over all the pkl files stores in the ts amazon bucket, download each one of them, load and upgrade the movie version to the latest, save the file and upload back to amazon bucket. Use this when something changes about the data format. """ logging.basicConfig(filename="upgrade.log", level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(thread)d PVM_run : %(message)s ') logging.getLogger().addHandler(logging.StreamHandler()) parser = argparse.ArgumentParser(description=doc) parser.add_argument("-u", "--upload", help="Actually upload the movies", action="store_true") parser.add_argument("-f", "--force", help="Force download", action="store_true") dsize = (480, 270) args = parser.parse_args() storage = PVM_Storage.Storage() dataset = PVM_datasets.PVMDataset(name="all") for [f, c] in dataset.all: print "Processing file %s" % f local_path = storage.get(f) # continue fc = lm.FrameCollection() fc_scaled = lm.FrameCollection() fc.load_from_file(local_path) for i in range(len(fc)): frame = fc.Frame(i) image = frame.get_image(channel=c) label = frame.get_label(channel=c) resized = cv2.resize(image, dsize=dsize, interpolation=cv2.INTER_CUBIC) label.scale_to_new_image_shape(new_image_shape=resized.shape, old_image_shape=image.shape) new_frame = lm.LabeledMovieFrame(internal_storage_method="jpg", compression_level=75) new_frame.set_image(resized, channel=c) new_frame.set_label(label=label, channel=c) fc_scaled.append(Frame=new_frame) local_saved = os.path.join("/tmp", os.path.basename(local_path)) remote_saved = os.path.join("PVM_data/", os.path.basename(local_path)) fc_scaled.write_to_file(local_saved) storage.put(path=remote_saved, from_path=local_saved)