[
  {
    "path": ".gitignore",
    "content": ".idea/\n*.pyc\n.cache/\nworkspace/\ndata\nloss.png\n*.egg-info\n\n"
  },
  {
    "path": "README.md",
    "content": "# shaplets\nPython implementation of [the Learning Time-Series Shapelets method by Josif Grabocka et al.](http://www.ismll.uni-hildesheim.de/pub/pdfs/grabocka2014e-kdd.pdf), that learns a shapelet-based time-series classifier with gradient descent. \n\nThis implementation views the model as a layered graph, where each layer \nimplements a forward, backword and parameters update methods (see below \ndiagram). This abstraction simplifies thinking about the algorithm and implementing it.\n![Network diagram](lts-diag.png)\n\n## Differences from the paper ##\n* This implmenetation employs two (LinearLayer + SigmoidLayer) pairs instead of one (LinearLayer + SigmoidLayer) pair as in the paper (and shown in above diagram). This (using two pairs) has yielded improved results on some datasets. To have a similar setup as in the paper, simply update `shapelets_lts/classification/shapelet_models.py:LtsShapeletClassifier._init_network()`. \n* The loss in this implementation is an updated version of the one in the \npaper to allow training a single model for all the classes in the dataset (rather than one model/class). The impact on performance was not analysed. For details check `shapelets_lts/network/cross_entropy_loss_layer.py`\n\n## Installation ##\n```bash\ngit clone git@github.com:mohaseeb/shaplets-python.git\ncd shaplets-python\npip install .\n# or, for dev\n# pip install .[dev]\n```\n## Usage ##\n```python\nfrom shapelets_lts.classification import LtsShapeletClassifier\n\n# create an LtsShapeletClassifier instance\nclassifier = LtsShapeletClassifier(\n    K=20,\n    R=3,\n    L_min=30,\n    epocs=50,\n    lamda=0.01,\n    eta=0.01,\n    shapelet_initialization='segments_centroids',\n    plot_loss=True\n)\n\n# train the classifier. \n# train_data.shape -> (# train samples X time-series length) \n# train_label.shape -> (# train samples)\nclassifier.fit(train_data, train_label, plot_loss=True)\n\n# evaluate on test data. \n# test_data.shape -> (# test samples X time-series length)\nprediction = classifier.predict(test_data)\n\n# retrieve the learnt shapelets\nshapelets = classifier.get_shapelets()\n\n\n# and plot sample shapelets\nfrom shapelets_lts.util import plot_sample_shapelets\nplot_sample_shapelets(shapelets=shapelets, sample_size=36)\n```\nAlso have a look at example.py. For a stable training, the samples might need to be scaled.\n\nExample plot from plot_sample_shapelets.\n![sample_shapelets](sample_shapelets.png)\n"
  },
  {
    "path": "example.py",
    "content": "from __future__ import division, print_function\n\nfrom os.path import expanduser\n\nfrom sklearn.metrics import classification_report\n\nfrom shapelets_lts.classification import LtsShapeletClassifier\nfrom shapelets_lts.util import ucr_dataset_loader, plot_sample_shapelets\n\n\"\"\"\nThis example uses dataset from the UCR archive \"UCR Time Series Classification\nArchive\" format.  \n\n- Follow the instruction on the UCR page \n(http://www.cs.ucr.edu/~eamonn/time_series_data/) to download the dataset. You \nneed to be patient! :) \n- Update the vars below to point to the correct dataset location in your  \nmachine.\n\nOtherwise update _load_train_test_datasets() below to return your own dataset.\n\"\"\"\n\nucr_dataset_base_folder = expanduser('~/ws/data/UCR_TS_Archive_2015/')\nucr_dataset_name = 'Gun_Point'\n\n\ndef main():\n    # load the data\n    print('\\nLoading data...')\n    x_train, y_train, x_test, y_test = _load_train_test_datasets()\n\n    # create a classifier\n    Q = x_train.shape[1]\n    K = int(0.15 * Q)\n    L_min = int(0.2 * Q)\n    clf = LtsShapeletClassifier(\n        K=K,\n        R=3,\n        L_min=L_min,\n        epocs=30,\n        lamda=0.01,\n        eta=0.01,\n        shapelet_initialization='segments_centroids',\n        plot_loss=True\n    )\n\n    # train the classifier\n    print('\\nTraining...')\n    clf.fit(x_train, y_train)\n\n    # evaluate on test data\n    print('\\nEvaluating...')\n    y_pred = clf.predict(x_test)\n    print(\n        'classification report...\\n{}'\n        ''.format(classification_report(y_true=y_test, y_pred=y_pred))\n    )\n\n    # plot sample shapelets\n    print('\\nPlotting sample shapelets...')\n    plot_sample_shapelets(shapelets=clf.get_shapelets(), sample_size=36)\n\n\ndef _load_train_test_datasets():\n    \"\"\"\n    :return: numpy arrays, train_data, train_labels, test_data, test_labels\n        train_data and test_data shape is: (n_samples, n_features)\n        train_labels and test_labels shape is: (n_samples)\n    \"\"\"\n    return ucr_dataset_loader.load_dataset(\n        dataset_name=ucr_dataset_name,\n        dataset_folder=ucr_dataset_base_folder\n    )\n\n\nif __name__ == '__main__':\n    main()\n"
  },
  {
    "path": "setup.py",
    "content": "from setuptools import setup, find_packages\n\nsetup(\n    name='shapelets-lts',\n    version='0.3.1',\n    install_requires=[\n        'numpy>=1.15.4,<2.0.0',\n        'pandas>=0.23.4,<2.0.0'\n        'scipy>=1.2.0,<2.0.0',\n        'scikit-learn>=0.20.2,<2.0.0',\n        'matplotlib>=2.2.3,<3.0.0',\n        'seaborn>=0.9.0,<2.0.0'\n    ],\n    extras_require={'dev': ['nose>=1.3.7,<2.0.0', 'ipython>=5.8.0,<6.0.0']},\n    packages=find_packages()\n)\n"
  },
  {
    "path": "shapelets_lts/__init__.py",
    "content": ""
  },
  {
    "path": "shapelets_lts/classification/__init__.py",
    "content": "from .shapelet_models import LtsShapeletClassifier\n"
  },
  {
    "path": "shapelets_lts/classification/shapelet_models.py",
    "content": "from __future__ import division\n\nimport copy\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.base import BaseEstimator\n\nfrom shapelets_lts.network import AggregationLayer\nfrom shapelets_lts.network import CrossEntropyLossLayer\nfrom shapelets_lts.network import LinearLayer\nfrom shapelets_lts.network import Network\nfrom shapelets_lts.network import SigmoidLayer\nfrom shapelets_lts.network import SoftMinLayer\nfrom shapelets_lts.util import utils\n\n\"\"\"\nThis class implements the sklearn estimator interface, so sklearn tools like GridsearchCV can be used\n\"\"\"\nclass LtsShapeletClassifier(BaseEstimator):\n    def __init__(self, K=20, R=3, L_min=30, alpha=-100, eta=0.01, lamda=0.01, epocs=10,\n                 shapelet_initialization='segments_centroids', plot_loss=False):\n        \"\"\"\n\n        :param K: number of shapelets\n        :param R: scales of shapelet lengths\n        :param L_min: minimum shapelet length\n        \"\"\"\n        # Shapelet related\n        self.K = K\n        self.R = R\n        self.n_shapelets = None\n        self.L_min = L_min\n        self.alpha = alpha\n        # Training data related\n        self.train_data = None\n        self.train_labels = None\n        self._orig_labels = None\n        self.output_size = None\n        self.train_size = None\n        # validation data\n        self.valid_data = None\n        self.valid_labels = None\n        # Hyper parameters\n        self.epocs = epocs\n        self.eta = eta  # learning rate\n        self.lamda = lamda  # regularization parameter\n        # other\n        self.network = None\n        self.shapelet_initialization = shapelet_initialization\n        self.plot_loss = plot_loss\n\n        self.loss_ax = self.valid_ax = None\n\n    def set_params(self, **parameters):\n        for parameter, value in parameters.items():\n            setattr(self, parameter, value)\n        return self\n\n    def fit(self, X, y):\n        self.n_shapelets = self.K * self.R\n        self.train_data = X\n        self._orig_labels = y\n        self.train_labels = utils.get_one_active_representation(y)\n        self.train_size, self.output_size = self.train_labels.shape\n        self._init_network()\n        self._train_network()\n        return self\n\n    def get_shapelets(self):\n        \"\"\"\n        Returns:\n            list: List of 1-d numpy arrays, one array per shapelet.\n        \"\"\"\n        # Each shapelet is contained in SoftMinLayer. SoftMinLayers are\n        # stored in AggregationLayer\n        [aggregation_layer] = [\n            layer for layer in self.network.get_layers()\n            if isinstance(layer, AggregationLayer)\n        ]\n\n        return aggregation_layer.get_shapelets()\n\n    def predict(self, X):\n        tmp_network = copy.deepcopy(self.network)\n        tmp_network.remove_loss_layer()\n        predicted_labels = np.zeros((X.shape[0], 1))\n        for i in range(X.shape[0]):\n            predicted_probabilities = tmp_network.forward(np.array([X[i, :]]), None)\n            predicted_labels[i, 0] = np.argmax(predicted_probabilities)\n        del tmp_network\n        return predicted_labels\n\n    def _init_network(self):\n        print('Network initialization ...')\n        self.network = Network()\n        # shapelets layer\n        self.network.add_layer(self._get_shapelets_layer())\n\n        # linear layer\n        self.network.add_layer(LinearLayer(self.n_shapelets, 24, self.eta, self.lamda, self.train_size),\n                               regularized=True)\n        # sigmoid layer\n        self.network.add_layer(SigmoidLayer(24))\n\n        # linear layer\n        self.network.add_layer(LinearLayer(24, self.output_size, self.eta, self.lamda, self.train_size),\n                               regularized=True)\n        # sigmoid layer\n        self.network.add_layer(SigmoidLayer(self.output_size))\n        # loss layer\n        self.network.add_layer(CrossEntropyLossLayer(self.lamda, self.train_size))\n\n    def _get_shapelets_layer(self):\n        if self.shapelet_initialization == 'segments_centroids':\n            print('Using training data to initialize shaplets')\n            return self._create_shapelets_layer_segments_centroids()\n        else:\n            print('Randomly initialize shapelets')\n            return self._create_shapelets_layer_random()\n\n    def _create_shapelets_layer_segments_centroids(self):\n        # Shapelets are included in SoftMinLayers\n        min_soft_layers = []\n        for r in range(1, self.R + 1):\n            L = r * self.L_min\n            top_K_centroids_scale_r = utils.get_centroids_of_segments(self.train_data, L, self.K)\n            for centroid in top_K_centroids_scale_r:\n                min_soft_layers.append(\n                    SoftMinLayer(np.array([centroid]), self.eta, self.alpha))\n        # shapelets aggregation layer\n        aggregator = AggregationLayer(min_soft_layers)\n        return aggregator\n\n    def _create_shapelets_layer_random(self):\n        # Shapelets are included in SoftMinLayers\n        min_soft_layers = []\n        for k in range(self.K):\n            for r in range(1, self.R + 1):\n                min_soft_layers.append(\n                    SoftMinLayer(np.random.normal(loc=0, scale=1, size=(1, r * self.L_min)), self.eta, self.alpha))\n        # shapelets aggregation layer\n        aggregator = AggregationLayer(min_soft_layers)\n        return aggregator\n\n    def _train_network(self):\n        print('Training ...')\n        loss = np.zeros((1, self.epocs * self.train_size))\n        valid_accur = np.zeros((1, self.epocs * self.train_size))\n\n        if self.valid_data is None:\n            print('Using training data for validation')\n            self.valid_data = self.train_data\n            self.valid_labels = self._orig_labels\n\n        iteration = 0\n        for epoc in range(self.epocs):\n            l = 10000\n            for sample_id in range(self.train_size):\n                sample = np.array([self.train_data[sample_id]])\n                label = np.array([self.train_labels[sample_id]])\n                # perform a forward pass\n                l = self.network.forward(sample, label)\n                # perform a backward pass\n                self.network.backward()\n                # perform a parameter update\n                self.network.update_params()\n                iteration += 1\n            loss[0, epoc] = l\n\n            # calculate accuracy in validation set\n            valid_epoc_accur = np.sum(\n                np.equal(\n                    self.predict(self.valid_data).ravel(),\n                    self.valid_labels\n                )\n            ) / self.valid_labels.shape[0]\n\n            valid_accur[0, epoc] = valid_epoc_accur\n            # print current loss info\n            print(\n                'epoch={}/{} (iteration={}) loss={} validation accuracy={}'\n                ''.format(epoc + 1, self.epocs, iteration, l, valid_epoc_accur)\n            )\n            # plot if needed\n            if self.plot_loss:\n                self._plot_loss(loss, valid_accur, epoc)\n        if self.plot_loss:\n            plt.savefig('loss.png')\n\n    def _plot_loss(self, loss, validation_acc, epocs):\n        if self.loss_ax is None:\n            _, self.loss_ax = plt.subplots(figsize=(10, 10))\n            self.valid_ax = self.loss_ax.twinx()\n            plt.xlabel(\"epoc\")\n            plt.ion()\n\n        self.loss_ax.plot(range(epocs + 1), loss[0, 0:epocs + 1], color='red')\n        self.loss_ax.set_ylabel('loss', color='red')\n        self.loss_ax.set_xlabel('epoc')\n        self.loss_ax.tick_params('y', colors='r')\n\n        self.valid_ax.plot(range(epocs + 1), validation_acc[0, 0:epocs + 1],\n                      color='blue')\n        self.valid_ax.set_ylabel('validation accuracy', color='blue')\n        self.valid_ax.tick_params('y', colors='b')\n\n        plt.pause(0.05)\n"
  },
  {
    "path": "shapelets_lts/network/__init__.py",
    "content": "from .linear_layer import LinearLayer\nfrom .aggregation_layer import AggregationLayer\nfrom .cross_entropy_loss_layer import CrossEntropyLossLayer\nfrom .sigmoid_layer import SigmoidLayer\nfrom .soft_min_layer import SoftMinLayer\nfrom .network import Network\n"
  },
  {
    "path": "shapelets_lts/network/aggregation_layer.py",
    "content": "from __future__ import print_function\nimport numpy as np\n\n\nclass AggregationLayer:\n    def __init__(self, layers):\n        self.layers = layers\n        self.layers_number = len(layers)\n        self.number_params = self._get_total_number_params()\n        # layer input holder\n        self.current_input = None\n        # layer output holder\n        self.current_output = None\n        # derivative of Loss w.r.t. inputs\n        self.dL_dinput = None\n\n    def _get_total_number_params(self):\n        total = 0\n        for layer_number in range(self.layers_number):\n            total += self.layers[layer_number].get_size()\n        return total\n\n    def forward(self, layer_input):\n        self.current_input = layer_input\n        self.current_output = np.zeros((1, self.layers_number))\n        for layer_number in range(self.layers_number):\n            self.current_output[0, layer_number] = self.layers[layer_number].forward(self.current_input)\n        return self.current_output\n\n    def backward(self, dL_dout):\n        dL_dparams = np.zeros((1, self.number_params))\n        layer_segment_id = 0\n        for layer_number in range(self.layers_number):\n            layer_size = self.layers[layer_number].get_size()\n            dL_dparams[0, layer_segment_id:layer_segment_id + layer_size] = self.layers[layer_number].backward(\n                dL_dout[0, layer_number])[:]\n            layer_segment_id += layer_size\n        return dL_dparams\n\n    def get_params(self):\n        \"\"\"\n\n        :return:\n        \"\"\"\n        params = np.zeros((1, self.number_params))\n        layer_segment_id = 0\n        for layer_number in range(self.layers_number):\n            layer_size = self.layers[layer_number].get_size()\n            params[0, layer_segment_id:layer_segment_id + layer_size] = self.layers[layer_number].get_params()[:]\n            layer_segment_id += layer_size\n        return params\n\n    def set_params(self, params):\n        \"\"\"\n\n        :param params:\n        :return:\n        \"\"\"\n        layer_segment_id = 0\n        for layer_number in range(self.layers_number):\n            layer_size = self.layers[layer_number].get_size()\n            self.layers[layer_number].set_params(params[0, layer_segment_id:layer_segment_id + layer_size])\n            layer_segment_id += layer_size\n\n    def update_params(self):\n        for layer_number in range(self.layers_number):\n            self.layers[layer_number].update_params()\n\n\n    def get_shapelets(self):\n        return [layer.get_shapelet() for layer in self.layers]"
  },
  {
    "path": "shapelets_lts/network/cross_entropy_loss_layer.py",
    "content": "from __future__ import print_function\nfrom __future__ import division\nimport numpy as np\n\n\nclass CrossEntropyLossLayer:\n    def __init__(self, lamda, train_size):\n        self.lamda = lamda\n        self.I = train_size\n        self.output_size = 1\n        # layer input holder\n        self.current_input_probabilities = None\n        # layer output holder\n        self.current_output = None\n        # derivative of Loss w.r.t. inputs\n        self.dL_dinput = None\n        # target probabilities\n        self.current_target_probabilities = None\n        # parameters to be penalized in the loss function\n        self.regularized_params = None\n\n    def set_current_target_probabilities(self, target_probabilities):\n        self.current_target_probabilities = target_probabilities\n\n    def set_regularized_params(self, regularized_params):\n        self.regularized_params = regularized_params\n\n    def forward(self, layer_input):\n        self.current_input_probabilities = layer_input\n        self.current_output = np.sum(\n            -1 * self.current_target_probabilities * np.log(self.current_input_probabilities) + (\n                self.current_target_probabilities - 1) * np.log(\n                1 - self.current_input_probabilities))\n        # regularization part\n        self.current_output += self.lamda * np.sum(self.regularized_params ** 2) / (\n            self.I * self.current_input_probabilities.size)\n        return self.current_output\n\n    def backward(self, dL_dout):\n        self.dL_dinput = -self.current_target_probabilities / self.current_input_probabilities + \\\n                         (1 - self.current_target_probabilities) / (1 - self.current_input_probabilities)\n        self.dL_dinput *= dL_dout\n        return self.dL_dinput\n\n    def update_params(self):\n        pass\n"
  },
  {
    "path": "shapelets_lts/network/linear_layer.py",
    "content": "from __future__ import print_function\nfrom __future__ import print_function\nfrom __future__ import print_function\nimport numpy as np\n\n\nclass LinearLayer:\n    def __init__(self, input_size, output_size, learning_rate, lamda, training_size):\n        \"\"\"\n\n        :param input_size:\n        :param output_size:\n        \"\"\"\n        self.input_size = input_size\n        self.output_size = output_size\n        # learning rate\n        self.eta = learning_rate\n        # regularization factor\n        self.lamda = lamda\n        # training data size\n        self.I = training_size\n        # layer weights\n        self.W = None\n        # layer biases\n        self.W_0 = None\n        self.set_weights(np.random.normal(loc=0, scale=1, size=(1, output_size * input_size)),\n                         np.random.normal(loc=0, scale=1, size=(1, output_size)))\n        # layer input holder\n        self.current_input = None\n        # layer output holder\n        self.current_output = None\n        # derivative of Loss w.r.t. inputs\n        self.dL_dinput = None\n        # derivative of Loss w.r.t. weights\n        self.dL_dparams = None\n\n    def set_weights(self, W, W_0):\n        self.W = W\n        self.W_0 = W_0\n\n    def forward(self, layer_input):\n        \"\"\"\n\n        :param layer_input:\n        :return:\n        \"\"\"\n        self.current_input = layer_input\n        self.current_output = np.dot(np.reshape(self.W, (self.output_size, self.input_size)),\n                                     self.current_input.T) + np.reshape(self.W_0, (self.output_size, 1))\n        return self.current_output.T\n\n    def backward(self, dL_dout):\n        \"\"\"\n\n        :param dL_dout: (1 X output_size)\n        :return: dL_dinputs (1 X input_size), dL_dW (output_size X input_size), dL_dW_0 (output_size, 1)\n        \"\"\"\n        # dL_dW calculations\n        dout_dparams = np.zeros((self.output_size, self.W.size + self.W_0.size))\n        output_W_index = 0\n        output_W_0_index = self.W.size\n        for output_id in range(self.output_size):\n            dout_dparams[output_id, output_W_index:output_W_index + self.input_size] = self.current_input\n            dout_dparams[output_id, output_W_0_index] = 1\n            output_W_index += self.input_size\n            output_W_0_index += 1\n        self.dL_dparams = np.dot(dL_dout, dout_dparams)\n        # dL_dinputs calculations\n        self.dL_dinput = np.dot(dL_dout, np.reshape(self.W, (self.output_size, self.input_size)))\n        return self.dL_dinput\n\n    def update_params(self):\n        \"\"\"\n\n        :param update_matrix:\n        :return:\n        \"\"\"\n        self.W -= self.eta * (\n            self.dL_dparams[0, 0:self.W.size] + 2 * self.lamda * self.W / (self.I * self.output_size))\n        self.W_0 -= self.eta * self.dL_dparams[0, self.W.size:]\n\n    def get_dL_dparams(self):\n        return self.dL_dparams\n\n    def get_params(self):\n        \"\"\"\n\n        :return:\n        \"\"\"\n        return np.concatenate((self.W.reshape((1, self.W.size)),\n                               self.W_0.reshape((1, self.W_0.size))), axis=1)\n\n    def set_params(self, params):\n        self.W = np.reshape(params[:, 0:params.size - self.W_0.size], (self.output_size, self.input_size))\n        self.W_0 = np.reshape(params[:, params.size - self.W_0.size:], (self.output_size, 1))\n"
  },
  {
    "path": "shapelets_lts/network/network.py",
    "content": "from . import CrossEntropyLossLayer\nimport numpy as np\n\n\nclass Network:\n    def __init__(self):\n        self.layers = []\n        self.regularized = []\n\n    def add_layer(self, layer, regularized=False):\n        self.layers.append(layer)\n        self.regularized.append(regularized)\n\n    def remove_loss_layer(self):\n        if isinstance(self.layers[-1], CrossEntropyLossLayer):\n            del self.layers[-1]\n\n    def forward(self, sample, target):\n        layer_input = sample\n        for layer_id in range(len(self.layers)):\n            if isinstance(self.layers[layer_id], CrossEntropyLossLayer):\n                self.layers[layer_id].set_current_target_probabilities(target)\n                self.layers[layer_id].set_regularized_params(self._get_regularized_params())\n            layer_input = self.layers[layer_id].forward(layer_input)\n        return layer_input\n\n    def backward(self):\n        dL_dlayer_output = 1\n        for layer_id in range(len(self.layers) - 1, -1, -1):\n            dL_dlayer_output = self.layers[layer_id].backward(dL_dlayer_output)\n\n    def update_params(self):\n        for layer_id in range(len(self.layers)):\n            self.layers[layer_id].update_params()\n\n    def get_layers(self):\n        return self.layers\n\n    def _get_regularized_params(self):\n        regularized = []\n        for layer_id in range(len(self.layers)):\n            if self.regularized[layer_id]:\n                regularized.append(self.layers[layer_id].get_params())\n        return np.concatenate(regularized, axis=1)\n"
  },
  {
    "path": "shapelets_lts/network/sigmoid_layer.py",
    "content": "from shapelets_lts.util import utils\n\n\nclass SigmoidLayer:\n    def __init__(self, input_size):\n        self.input_size = input_size\n        self.output_size = input_size\n        # layer input holder\n        self.current_input = None\n        # layer output holder\n        self.current_output = None\n        # derivative of Loss w.r.t. inputs\n        self.dL_dinput = None\n\n    def forward(self, layer_input):\n        self.current_input = layer_input\n        self.current_output = utils.sigmoid(self.current_input)\n        return self.current_output\n\n    def backward(self, dL_dout):\n        dout_dinput = self.current_output * (1 - self.current_output)\n        self.dL_dinput = dL_dout * dout_dinput\n        return self.dL_dinput\n\n    def update_params(self):\n        pass\n"
  },
  {
    "path": "shapelets_lts/network/soft_min_layer.py",
    "content": "import numpy as np\n\n\nclass SoftMinLayer:\n    def __init__(self, sequence, learning_rate=0.01, alpha=-100):\n        \"\"\"\n\n        :type alpha:\n        :param sequence:\n        :param alpha:\n        \"\"\"\n        self.S = sequence\n        self.L = np.size(sequence, 1)\n        self.alpha = alpha\n        # learning rate\n        self.eta = learning_rate\n        # layer input holder\n        self.T = None\n        # layer output holder\n        self.current_output = None\n        # derivative of Loss w.r.t. shapelet values\n        self.dL_dS = None\n        # holder of pre-calculated values to speed up the calculations\n        self.J = None  # number of segments in input time-series\n        self.D = None  # (1 X J) distances between shapelet and the current time-series segments\n        self.xi = None  # (1 X J)\n        self.psi = None\n        self.M = None  # soft minimum distance\n\n    def forward(self, layer_input):\n        self.T = layer_input\n        self.M = self.dist_soft_min()\n        return self.M\n\n    def backward(self, dL_dout):\n        \"\"\"\n\n        :param dL_dout:\n        :return: dL_dS (1 X self.L)\n        \"\"\"\n        # (1 X J): derivative of M (soft minimum) w.r.t D_j (distance between shapelet and the segment j of the\n        # time-series)\n        dM_dD = self.xi * (1 + self.alpha * (self.D - self.M)) / self.psi\n        # (J X L) : derivative of D_j w.r.t. S_l (shapelet value at position l)\n        dD_dS = np.zeros((self.J, self.L))\n        for j in range(self.J):\n            dD_dS[j, :] = 2 * (self.S - self.T[0, j:j + self.L]) / self.L\n        # (1 X L) : derivative of M w.r.t. S_l\n        dM_dS = np.dot(dM_dD, dD_dS)\n        # (1 X L) : derivative of L w.r.t S_l. Note dL_dout is dL_dM\n        self.dL_dS = dL_dout * dM_dS\n        return self.dL_dS\n\n    def dist_soft_min(self):\n        Q = self.T.size\n        self.J = Q - self.L + 1\n        M_numerator = 0\n        # for each segment of T\n        self.D = np.zeros((1, self.J))\n        self.xi = np.zeros((1, self.J))\n        self.psi = 0\n        for j in range(self.J):\n            self.D[0, j] = self.dist_sqr_error(self.T[0, j:j + self.L])\n            self.xi[0, j] = np.exp(self.alpha * self.D[0, j])\n            M_numerator += self.D[0, j] * self.xi[0, j]\n            self.psi += self.xi[0, j]\n        M = M_numerator / self.psi\n        return M\n\n    def dist_sqr_error(self, T_j):\n        \"\"\"\n\n        :param T:\n        :return:\n        \"\"\"\n        dist = (T_j - self.S) ** 2\n        dist = np.sum(dist) / self.L\n        return dist\n\n    def get_params(self):\n        \"\"\"\n\n        :return:\n        \"\"\"\n        return self.S\n\n    def set_params(self, param):\n        \"\"\"\n\n        :param param:\n        :return:\n        \"\"\"\n        self.S = param\n\n    def update_params(self):\n        self.S -= self.eta * self.dL_dS\n\n    def get_size(self):\n        return self.L\n\n    def get_shapelet(self):\n        return self.S.ravel()\n"
  },
  {
    "path": "shapelets_lts/tests/test_aggregation_layer.py",
    "content": "import numpy as np\n\nfrom shapelets_lts.network import AggregationLayer\nfrom shapelets_lts.util import utils, soft_min_layer_factory\n\n\ndef test_forward():\n    # create a set of soft_min_layers\n    soft_min_layers = soft_min_layer_factory.create_soft_min_layers([3, 4, 5])\n    # create an aggregation\n    aggregator = AggregationLayer(soft_min_layers)\n    # create a layer input\n    Q = 10\n    T = np.random.normal(loc=0, scale=1, size=(1, Q))\n    # do a forward pass\n    output = aggregator.forward(T)\n    # compare to the truth output\n    output_truth = np.array([[layer.forward(T) for layer in soft_min_layers]])\n    assert (np.array_equal(output, output_truth))\n\n\ndef test_backward():\n    layer_sizes = [3, 4, 5]\n    n_outputs = len(layer_sizes)\n    # create a set of soft_min_layers\n    soft_min_layers = soft_min_layer_factory.create_soft_min_layers(layer_sizes)\n    # create an aggregation\n    aggregator = AggregationLayer(soft_min_layers)\n    # create a layer input\n    Q = 10\n    T = np.random.normal(loc=0, scale=1, size=(1, Q))\n    # create a dL_dout\n    dL_dout = np.random.normal(loc=0, scale=1, size=(1, n_outputs))\n    # do a forward and backward passes\n    aggregator.forward(T)\n    dL_dparams = aggregator.backward(dL_dout)\n    # verify dL_dS ######\n    dout_dparams_truth = utils.approximate_derivative_wrt_params(aggregator, T, n_outputs, h=0.00001)\n    dL_dparams_truth = np.dot(dL_dout, dout_dparams_truth)\n    result = np.isclose(dL_dparams, dL_dparams_truth, rtol=1e-05, atol=1e-03)\n    assert result.all()\n"
  },
  {
    "path": "shapelets_lts/tests/test_cross_entropy_loss_layer.py",
    "content": "import numpy as np\n\nfrom shapelets_lts.network import CrossEntropyLossLayer\nfrom shapelets_lts.util import utils\n\n\ndef test_forward():\n    training_data_size = 10\n    lamda = 0.01\n    params = np.array([[1, 2, 3, 4]])\n    # create a cross entropy layer\n    layer = CrossEntropyLossLayer(lamda, training_data_size)\n    # create a layer input,\n    input_probabilities = np.array([0.1, 0.4, 0.5])\n    target_probabilities = np.array([0, 0, 1])\n    # execute the layer\n    layer.set_current_target_probabilities(target_probabilities)\n    layer.set_regularized_params(params)\n    layer_output = layer.forward(input_probabilities)\n    # compare the layer output to the expected one\n    output_truth = 1.30933331998  # calculated by hand\n    assert (layer_output, output_truth)\n\n\ndef test_backward():\n    training_data_size = 10\n    lamda = 0.01\n    params = np.array([[1, 2, 3, 4]])\n    # create a layer\n    layer = CrossEntropyLossLayer(lamda, training_data_size)\n    # create a layer input,\n    input_probabilities = np.array([[0.7, 0.4, 0.5, 0.1]])\n    target_probabilities = np.array([[0, 1, 0, 0]])\n    # create dL_layer_doutput\n    dL_doutput = 1\n    # perform a forward and a backward pass\n    layer.set_current_target_probabilities(target_probabilities)\n    layer.set_regularized_params(params)\n    layer.forward(input_probabilities)\n    dL_dinput = layer.backward(dL_doutput)\n    # verify dL_dinput ######\n    doutput_dinput_truth = utils.approximate_derivative_wrt_inputs(layer.forward, input_probabilities, 1,\n                                                                   h=0.000001)  # n_outputs X n_inputs\n    dL_dinput_truth = np.dot(dL_doutput, doutput_dinput_truth)\n    result = np.isclose(dL_dinput, dL_dinput_truth)\n    assert result.all()\n"
  },
  {
    "path": "shapelets_lts/tests/test_linear_layer.py",
    "content": "from __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\n\nfrom shapelets_lts.network import LinearLayer\nfrom shapelets_lts.util import utils\n\n\ndef test_fc_layer_initialization():\n    n_inputs = 15\n    n_outputs = 4\n    learning_rate = 0.01\n    regularization_parameter = 0.1\n    training_size = 10\n    fc_layer = LinearLayer(n_inputs, n_outputs, learning_rate, regularization_parameter, training_size)\n    W = fc_layer.get_params()\n    assert (W.shape == (1, n_outputs * (n_inputs + 1)))\n\n\ndef test_forward():\n    n_inputs = 15\n    n_outputs = 4\n    learning_rate = 0.01\n    regularization_parameter = 0.1\n    training_size = 10\n    fc_layer = LinearLayer(n_inputs, n_outputs, learning_rate, regularization_parameter, training_size)\n    # initialize weights\n    W = np.ones((n_outputs, n_inputs))\n    W_0 = np.ones((n_outputs, 1))\n    fc_layer.set_weights(W, W_0)\n    # create a layer input\n    layer_input = np.ones((1, n_inputs))\n    # execute the layer\n    layer_output = fc_layer.forward(layer_input)\n    # compare the layer output to the expected one\n    expected_output = np.array([[16., 16., 16., 16.]])\n    assert (np.array_equal(layer_output, expected_output))\n\n\ndef test_backword():\n    \"\"\"\n    :return:\n    \"\"\"\n    # create a layer\n    n_inputs = 3\n    n_outputs = 2\n    learning_rate = 0.01\n    regularization_parameter = 0.1\n    training_size = 10\n    fc_layer = LinearLayer(n_inputs, n_outputs, learning_rate, regularization_parameter, training_size)\n    # create a layer input\n    layer_input = np.random.normal(loc=0, scale=1, size=(1, n_inputs))\n    # create dL_layer_doutput\n    dL_layer_output = np.random.normal(loc=0, scale=1, size=(1, n_outputs))\n    # do a forward and a backward pass\n    fc_layer.forward(layer_input)\n    dL_input = fc_layer.backward(dL_layer_output)\n    dL_dparams = fc_layer.get_dL_dparams()\n    # verify dL_dinput ######\n    doutput_input_truth = utils.approximate_derivative_wrt_inputs(fc_layer.forward, layer_input, n_outputs,\n                                                                  h=0.01)  # n_outputs X n_inputs\n    dL_input_truth = np.dot(dL_layer_output, doutput_input_truth)\n    result = np.isclose(dL_input, dL_input_truth)\n    assert result.all()\n    # verify dL_dW\n    dout_dparams_truth = utils.approximate_derivative_wrt_params(fc_layer, layer_input, n_outputs, h=0.0001)\n    dL_dparams_truth = np.dot(dL_layer_output, dout_dparams_truth)\n    result = np.isclose(dL_dparams, dL_dparams_truth, rtol=1e-05, atol=1e-03)\n    assert result.all()\n"
  },
  {
    "path": "shapelets_lts/tests/test_sigmoid_layer.py",
    "content": "from scipy.special import expit\n\nimport numpy as np\n\nfrom shapelets_lts.network import SigmoidLayer\nfrom shapelets_lts.util import utils\n\n\ndef test_forward():\n    n_inputs = 5\n    # create a layer\n    sig_layer = SigmoidLayer(n_inputs)\n    # create an input\n    layer_input = np.zeros((1, n_inputs)) + 0.458\n    output = sig_layer.forward(layer_input)\n    # compare to expected output\n    output_truth = np.zeros((1, n_inputs)) + expit(0.458)\n    assert (np.isclose(output, output_truth).all())\n\n\ndef test_backward():\n    n_inputs = 5\n    # create a layer\n    sig_layer = SigmoidLayer(n_inputs)\n    # create a layer input\n    layer_input = np.random.normal(loc=0, scale=1, size=(1, n_inputs))\n    # create dL_layer_doutput\n    dL_doutput = np.random.normal(loc=0, scale=1, size=(1, n_inputs))\n    # perform a forward and a backward pass\n    sig_layer.forward(layer_input)\n    dL_dinput = sig_layer.backward(dL_doutput)\n    # verify dL_dinput ######\n    doutput_dinput_truth = utils.approximate_derivative_wrt_inputs(sig_layer.forward, layer_input, n_inputs,\n                                                                   h=0.00001)  # n_outputs X n_inputs\n    dL_dinput_truth = np.dot(dL_doutput, doutput_dinput_truth)\n    result = np.isclose(dL_dinput, dL_dinput_truth)\n    assert result.all()\n"
  },
  {
    "path": "shapelets_lts/tests/test_soft_min_layer.py",
    "content": "import numpy as np\n\nfrom shapelets_lts.network import SoftMinLayer\nfrom shapelets_lts.util import utils\n\n\ndef test_forward():\n    soft_layer = SoftMinLayer(np.ones((1, 5)))\n    T = np.ones((1, 10)) + 1\n    assert (soft_layer.forward(T) == 1)\n\n\ndef test_backward():\n    # create the soft min layer\n    L = 5\n    shapelet = np.random.normal(loc=0, scale=1, size=(1, L))\n    soft_layer = SoftMinLayer(shapelet)\n    # create a layer input\n    Q = 10\n    T = np.random.normal(loc=0, scale=1, size=(1, Q))\n    # create dL_dM\n    dL_dM = np.random.normal()\n    # do a forward and a backward pass\n    soft_layer.forward(T)\n    dL_dS = soft_layer.backward(dL_dM)\n    # verify dL_dS ######\n    dM_dS_truth = utils.approximate_derivative_wrt_params(soft_layer, T, 1, h=0.00001)\n    dL_dS_truth = dL_dM * dM_dS_truth\n    result = np.isclose(dL_dS, dL_dS_truth, rtol=1e-05, atol=1e-04)\n    assert result.all()\n\n\ndef test_shapelet_dist_sqr_error():\n    soft_layer = SoftMinLayer(np.ones((1, 10)))\n    assert (soft_layer.dist_sqr_error(np.zeros((1, 10))) == 1)\n"
  },
  {
    "path": "shapelets_lts/tests/test_utils.py",
    "content": "import numpy as np\n\nfrom shapelets_lts.util import utils\n\n\ndef test_get_centroids():\n    cluster_size = 5\n    n_dims = 2\n    n_clusters = 3\n    cluster_1_data = np.random.normal(loc=0, scale=1, size=(cluster_size, n_dims))\n    cluster_2_data = np.random.normal(loc=5, scale=1, size=(cluster_size, n_dims))\n    cluster_3_data = np.random.normal(loc=9, scale=1, size=(cluster_size, n_dims))\n    data = np.concatenate((cluster_1_data, cluster_2_data, cluster_3_data), axis=0)\n    centroids = utils.get_centroids(data, n_clusters)\n    assert (centroids.shape == (n_clusters, n_dims))\n\n\ndef test_segment_dataset():\n    I = 2  # number of time-series\n    Q = 4  # time series size\n    L = 2  # segment length\n    J = Q - L + 1  # segments per time-series\n    data = np.random.normal(loc=5, scale=1, size=(I, Q))\n    S = utils.segment_dataset(data, 2)  # segment\n    assert (S.shape == (J * I, L))\n    assert (np.array_equal(S[I * J - 1], data[I - 1, Q - L:]))\n\n\ndef test_get_centroids_of_segments():\n    n_samples = 50\n    n_dims = 2\n    n_clusters = 2\n    cluster_1_data = np.random.normal(loc=0, scale=0.01, size=(n_samples, n_dims))\n    cluster_2_data = np.random.normal(loc=5, scale=0.01, size=(n_samples, n_dims))\n    data = np.concatenate((cluster_1_data, cluster_2_data), axis=1)\n    centroids = utils.get_centroids_of_segments(data, n_dims, n_clusters + 1)\n    # centroids should be close to (0,0) (0,5) (5,5)\n    centroid1_occur = 0\n    centroid2_occur = 0\n    centroid3_occur = 0\n    for centroid in centroids:\n        if np.isclose(centroid, (0, 0), rtol=1e-05, atol=1e-02).all():\n            centroid1_occur += 1\n        if np.isclose(centroid, (0, 5), rtol=1e-05, atol=1e-02).all():\n            centroid2_occur += 1\n        if np.isclose(centroid, (5, 5), rtol=1e-05, atol=1e-02).all():\n            centroid3_occur += 1\n    assert (centroid1_occur == centroid2_occur == centroid3_occur == 1)\n"
  },
  {
    "path": "shapelets_lts/util/__init__.py",
    "content": "from __future__ import division, print_function\n\nfrom .plotting import plot_sample_shapelets\n"
  },
  {
    "path": "shapelets_lts/util/plotting.py",
    "content": "from __future__ import division, print_function\n\nfrom random import sample\n\nimport pandas as pd\nimport seaborn as sns\n\n\ndef plot_sample_shapelets(shapelets, sample_size=1e6):\n    \"\"\"Plots a random sample from the passed shapelets.\n\n    Args:\n        shapelets (list): list of 1-d numpy arrays, one for each shapelet.\n        sample_size: number of random shapelets to be selected for plotting.\n    \"\"\"\n    # select maximum of len(shapelets) sample shapelets\n    some_shapelets = sample(shapelets, min(sample_size, len(shapelets)))\n\n    # put the shapelets in a format suitable for plotting with seaborn\n    def _shapelet_to_df(id_, shapelet):\n        return pd.DataFrame(\n            dict(shapelet=id_, X=range(len(shapelet)), Y=shapelet)\n        )\n\n    shapelets_df = pd.concat(\n        objs=[_shapelet_to_df(_id, s) for _id, s in enumerate(some_shapelets)],\n        axis=0\n    ).reset_index(drop=True)\n\n    # plot the shapelets\n    grid = sns.FacetGrid(shapelets_df, col=\"shapelet\", col_wrap=6)\n    grid.map(sns.lineplot, \"X\", \"Y\")\n"
  },
  {
    "path": "shapelets_lts/util/soft_min_layer_factory.py",
    "content": "import numpy as np\n\nfrom shapelets_lts.network.soft_min_layer import SoftMinLayer\n\n\ndef create_soft_min_layers(sizes):\n    layers = []\n    for size in sizes:\n        layers.append(SoftMinLayer(np.ones((1, size))))\n    return layers\n"
  },
  {
    "path": "shapelets_lts/util/ucr_dataset_loader.py",
    "content": "from os import path\n\nfrom numpy import genfromtxt\n\n\ndef load_dataset(dataset_name, dataset_folder):\n    dataset_path = path.join(dataset_folder, dataset_name)\n    train_file_path = path.join(dataset_path, '{}_TRAIN'.format(dataset_name))\n    test_file_path = path.join(dataset_path, '{}_TEST'.format(dataset_name))\n\n    # training data\n    train_raw_arr = genfromtxt(train_file_path, delimiter=',')\n    train_data = train_raw_arr[:, 1:]\n    train_labels = train_raw_arr[:, 0] - 1\n    # one was subtracted to change the labels to 0 and 1 instead of 1 and 2\n\n    # test_data\n    test_raw_arr = genfromtxt(test_file_path, delimiter=',')\n    test_data = test_raw_arr[:, 1:]\n    test_labels = test_raw_arr[:, 0] - 1\n\n    return train_data, train_labels, test_data, test_labels\n"
  },
  {
    "path": "shapelets_lts/util/utils.py",
    "content": "from __future__ import print_function\nimport numpy as np\nimport math\nfrom sklearn.cluster import KMeans\n\n\ndef approximate_derivative_wrt_inputs(function, inputs, n_outputs, h):\n    \"\"\"\n    https://en.wikipedia.org/wiki/Finite_difference#Relation_with_derivatives\n    :param function:\n    :param inputs:\n    :param n_outputs:\n    :param h:\n    :return:\n    \"\"\"\n    n_inputs = inputs.size\n    dFunction_dinputs = np.zeros((n_outputs, n_inputs))\n    for input_id in range(n_inputs):\n        f1 = function(inputs)  # 1 X n_outputs\n        inputs[0, input_id] += h\n        f2 = function(inputs)  # 1 X n_outputs\n        inputs[0, input_id] -= h\n        dFunction_dinputs[:, input_id] = (f2 - f1) / h\n    return dFunction_dinputs\n\n\ndef approximate_derivative_wrt_params(layer, inputs, n_outputs, h):\n    \"\"\"\n    https://en.wikipedia.org/wiki/Finite_difference#Relation_with_derivatives\n    :param n_outputs:\n    :param layer:\n    :param inputs:\n    :param h:\n    :return:\n    \"\"\"\n    params = layer.get_params()\n    n_params = params.size\n    dlayer_dparams = np.zeros((n_outputs, n_params))\n    for param_id in range(n_params):\n        f1 = layer.forward(inputs)  # 1 X n_outputs\n        params[0, param_id] += h\n        layer.set_params(params)\n        f2 = layer.forward(inputs)  # 1 X n_outputs\n        params[0, param_id] -= h\n        layer.set_params(params)\n        dlayer_dparams[:, param_id] = (f2 - f1) / h\n    return dlayer_dparams\n\n\ndef sigmoid(X):\n    return np.array([[(1 / (1 + math.exp(-x))) for x in X[0, :]]])\n\n\ndef get_one_active_representation(labels):\n    classes = np.unique(labels)\n    one_active_labels = np.zeros((labels.size, classes.size))\n    for label_id in range(labels.size):\n        one_active_labels[label_id, np.where(classes == labels[label_id])] = 1\n    return one_active_labels\n\n\ndef get_centroids_of_segments(data, L, K):\n    \"\"\"\n\n    :param data: the dataset\n    :param L: segment length\n    :param K: number of centroids\n    :return: the top K centroids of the clustered segments\n    \"\"\"\n    data_segmented = segment_dataset(data, L)\n    centroids = get_centroids(data_segmented, K)\n    return centroids\n\n\ndef segment_dataset(data, L):\n    \"\"\"\n\n    :param data:\n    :param L: segment length\n    :return:\n    \"\"\"\n    # number of time series, time series size\n    I, Q = data.shape\n    # number of segments in a time series\n    J = Q - L + 1\n    S = np.zeros((J * I, L))\n    # create segments\n    for i in range(I):\n        for j in range(J):\n            S[i * J + j, :] = data[i, j:j + L]\n    return S\n\n\ndef get_centroids(data, k):\n    clusterer = KMeans(n_clusters=k)\n    clusterer.fit(data)\n    return clusterer.cluster_centers_\n"
  }
]