[
  {
    "path": ".github/workflows/python-publish-on-release.yml",
    "content": "\nname: Python release on pypi\n\n\non:\n  release:\n    types: [published]\n  workflow_dispatch:\n\n\n\njobs:\n  build:\n\n    runs-on: ubuntu-20.04\n    steps:\n      - uses: actions/checkout@v3\n      - uses: actions/setup-python@v3\n        with:\n          python-version: \"3.10\"\n      - name: Assemble python package folder\n        run: |\n            mv model/ vae_anomaly_detection/\n      - name: Install pypa/hatch\n        run: pip install hatch\n      - name: Build a binary wheel and a source tarball\n        run: hatch build\n      - name: Publish distribution 📦 to PyPI\n        run: hatch publish\n        env:\n          HATCH_INDEX_USER: __token__\n          HATCH_INDEX_AUTH: ${{ secrets.PYPI_PASSWORD }}\n"
  },
  {
    "path": ".github/workflows/python-test-build.yml",
    "content": "\nname: Python test and build\n\non:\n  push:\n    tags-ignore:\n      - \"*\"\n  schedule:\n    - cron: \"0 0 * * 0\" # Run every Sunday at midnight\n  workflow_dispatch: # Manually trigger a workflow run\n    \n\njobs:\n  ci:\n    strategy:\n      fail-fast: false\n      matrix:\n        python-version: [\"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.11\"]\n        os: [ubuntu-20.04]\n    runs-on: ${{ matrix.os }}\n    env:\n      HATCH_ENV: test\n    steps:\n      - uses: actions/checkout@v2\n      - uses: actions/setup-python@v2\n        with:\n          python-version: ${{ matrix.python-version }}\n      - name: Install Hatch 🥚\n        run: pip install hatch\n      - name: Install dependencies\n        run: hatch env create test\n      -  name: Test with pytest\n         run: hatch run test:pytest\n      - name: rename folder\n        run: mv model/ vae_anomaly_detection/\n      - name: Build package 📦\n        run: hatch build"
  },
  {
    "path": ".gitignore",
    "content": "envs/\n.vscode/\n.idea\ndist/\npoetry.lock\n\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n#  Usually these files are written by a python script from a template\n#  before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n#   For a library or package, you might want to ignore these files since the code is\n#   intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n#   However, in case of collaboration, if having platform-specific dependencies or dependencies\n#   having no cross-platform support, pipenv may install dependencies that don't work, or not\n#   install all needed dependencies.\n#Pipfile.lock\n\n# poetry\n#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.\n#   This is especially recommended for binary packages to ensure reproducibility, and is more\n#   commonly ignored for libraries.\n#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control\n#poetry.lock\n\n# pdm\n#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.\n#pdm.lock\n#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it\n#   in version control.\n#   https://pdm.fming.dev/#use-with-ide\n.pdm.toml\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can\n#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n#  and can be added to the global gitignore or merged into this file.  For a more nuclear\n#  option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n"
  },
  {
    "path": ".projectignore",
    "content": "# This file contains a list of match patterns that instructs\n# anaconda-project to exclude certain files or directories when\n# building a project archive. The file format is a simplfied\n# version of Git's .gitignore file format. In fact, if the\n# project is hosted in a Git repository, these patterns can be\n# merged into the .gitignore file and this file removed.\n# See the anaconda-project documentation for more details.\n\n# Python caching\n*.pyc\n*.pyd\n*.pyo\n__pycache__/\n\n# Jupyter & Spyder stuff\n.ipynb_checkpoints/\n.Trash-*/\n/.spyderproject\n"
  },
  {
    "path": "dataset.py",
    "content": "from typing import Tuple\nimport torch\nfrom torch.utils.data import Dataset, TensorDataset\nfrom torchvision.datasets import MNIST\n\ndef rand_dataset(num_rows=60_000, num_columns=100) -> Dataset:\n    return TensorDataset(torch.rand(num_rows, num_columns))\n\n\ndef mnist_dataset(train=True) -> Dataset:\n    \"\"\"\n    Returns the MNIST dataset for training or testing.\n    \n    Args:\n    train (bool): If True, returns the training dataset. Otherwise, returns the testing dataset.\n    \n    Returns:\n    Dataset: The MNIST dataset.\n    \"\"\"\n    return MNIST(root='./data', train=train, download=True, transform=None)\n"
  },
  {
    "path": "model/VAE.py",
    "content": "from abc import abstractmethod, ABC\n\nimport torch\nfrom torch import nn\nfrom torch.distributions import Normal, kl_divergence\nfrom torch.nn.functional import softplus\nimport pytorch_lightning as pl\n\n\nclass VAEAnomalyDetection(pl.LightningModule, ABC):\n    \"\"\"\n    Variational Autoencoder (VAE) for anomaly detection. The model learns a low-dimensional representation of the input\n    data using an encoder-decoder architecture, and uses the learned representation to detect anomalies.\n\n    The model is trained to minimize the Kullback-Leibler (KL) divergence between the learned distribution of the latent\n    variables and the prior distribution (a standard normal distribution). It is also trained to maximize the likelihood\n    of the input data under the learned distribution.\n\n    This implementation uses PyTorch Lightning to simplify training and improve reproducibility.\n    \"\"\"\n\n    def __init__(self, input_size: int, latent_size: int, L: int = 10, lr: float = 1e-3, log_steps: int = 1_000):\n        \"\"\"\n        Initializes the VAEAnomalyDetection model.\n\n        Args:\n            input_size (int): Number of input features.\n            latent_size (int): Size of the latent space.\n            L (int, optional): Number of samples in the latent space to detect the anomaly. Defaults to 10.\n            lr (float, optional): Learning rate. Defaults to 1e-3.\n            log_steps (int, optional): Number of steps between each logging. Defaults to 1_000.\n        \"\"\"\n        super().__init__()\n        self.L = L\n        self.lr = lr\n        self.input_size = input_size\n        self.latent_size = latent_size\n        self.encoder = self.make_encoder(input_size, latent_size)\n        self.decoder = self.make_decoder(latent_size, input_size)\n        self.prior = Normal(0, 1)\n        self.log_steps = log_steps\n\n    @abstractmethod\n    def make_encoder(self, input_size: int, latent_size: int) -> nn.Module:\n        \"\"\"\n        Abstract method to create the encoder network.\n\n        Args:\n            input_size (int): Number of input features.\n            latent_size (int): Size of the latent space.\n\n        Returns:\n            nn.Module: Encoder network.\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def make_decoder(self, latent_size: int, output_size: int) -> nn.Module:\n        \"\"\"\n        Abstract method to create the decoder network.\n\n        Args:\n            latent_size (int): Size of the latent space.\n            output_size (int): Number of output features.\n\n        Returns:\n            nn.Module: Decoder network.\n        \"\"\"\n        pass\n\n    def forward(self, x: torch.Tensor) -> dict:\n        \"\"\"\n        Computes the forward pass of the model and returns the loss and other relevant information.\n\n        Args:\n            x (torch.Tensor): Input data. Shape [batch_size, num_features].\n\n        Returns:\n            Dictionary containing:\n            - loss: Total loss.\n            - kl: KL-divergence loss.\n            - recon_loss: Reconstruction loss.\n            - recon_mu: Mean of the reconstructed input.\n            - recon_sigma: Standard deviation of the reconstructed input.\n            - latent_dist: Distribution of the latent space.\n            - latent_mu: Mean of the latent space.\n            - latent_sigma: Standard deviation of the latent space.\n            - z: Sampled latent space.\n\n        \"\"\"\n        pred_result = self.predict(x)\n        x = x.unsqueeze(0)  # unsqueeze to broadcast input across sample dimension (L)\n        log_lik = Normal(pred_result['recon_mu'], pred_result['recon_sigma']).log_prob(x).mean(\n            dim=0)  # average over sample dimension\n        log_lik = log_lik.mean(dim=0).sum()\n        kl = kl_divergence(pred_result['latent_dist'], self.prior).mean(dim=0).sum()\n        loss = kl - log_lik\n        return dict(loss=loss, kl=kl, recon_loss=log_lik, **pred_result)\n\n    def predict(self, x) -> dict:\n        \"\"\"\n        Compute the output of the VAE. Does not compute the loss compared to the forward method.\n\n        Args:\n            x: Input tensor of shape [batch_size, input_size].\n\n        Returns:\n            Dictionary containing:\n            - latent_dist: Distribution of the latent space.\n            - latent_mu: Mean of the latent space.\n            - latent_sigma: Standard deviation of the latent space.\n            - recon_mu: Mean of the reconstructed input.\n            - recon_sigma: Standard deviation of the reconstructed input.\n            - z: Sampled latent space.\n\n        \"\"\"\n        batch_size = len(x)\n        latent_mu, latent_sigma = self.encoder(x).chunk(2, dim=1) #both with size [batch_size, latent_size]\n        latent_sigma = softplus(latent_sigma)\n        dist = Normal(latent_mu, latent_sigma)\n        z = dist.rsample([self.L])  # shape: [L, batch_size, latent_size]\n        z = z.view(self.L * batch_size, self.latent_size)\n        recon_mu, recon_sigma = self.decoder(z).chunk(2, dim=1)\n        recon_sigma = softplus(recon_sigma)\n        recon_mu = recon_mu.view(self.L, *x.shape)\n        recon_sigma = recon_sigma.view(self.L, *x.shape)\n        return dict(latent_dist=dist, latent_mu=latent_mu,\n                    latent_sigma=latent_sigma, recon_mu=recon_mu,\n                    recon_sigma=recon_sigma, z=z)\n\n    def is_anomaly(self, x: torch.Tensor, alpha: float = 0.05) -> torch.Tensor:\n        \"\"\"\n        Determines if input samples are anomalous based on a given threshold.\n        \n        Args:\n            x: Input tensor of shape (batch_size, num_features).\n            alpha: Anomaly threshold. Values with probability lower than alpha are considered anomalous.\n        \n        Returns:\n            A binary tensor of shape (batch_size,) where `True` represents an anomalous sample and `False` represents a \n            normal sample.\n        \"\"\"\n        p = self.reconstructed_probability(x)\n        return p < alpha\n\n    def reconstructed_probability(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Computes the probability density of the input samples under the learned\n        distribution of reconstructed data.\n\n        Args:\n            x: Input data tensor of shape (batch_size, num_features).\n\n        Returns:\n            A tensor of shape (batch_size,) containing the probability densities of\n            the input samples under the learned distribution of reconstructed data.\n        \"\"\"\n        with torch.no_grad():\n            pred = self.predict(x)\n        recon_dist = Normal(pred['recon_mu'], pred['recon_sigma'])\n        x = x.unsqueeze(0)\n        p = recon_dist.log_prob(x).exp().mean(dim=0).mean(dim=-1)  # vector of shape [batch_size]\n        return p\n\n    def generate(self, batch_size: int = 1) -> torch.Tensor:\n        \"\"\"\n        Generates a batch of samples from the learned prior distribution.\n\n        Args:\n            batch_size: Number of samples to generate.\n\n        Returns:\n            A tensor of shape (batch_size, num_features) containing the generated\n            samples.\n        \"\"\"\n        z = self.prior.sample((batch_size, self.latent_size))\n        recon_mu, recon_sigma = self.decoder(z).chunk(2, dim=1)\n        recon_sigma = softplus(recon_sigma)\n        return recon_mu + recon_sigma * torch.rand_like(recon_sigma)\n    \n    \n    def training_step(self, batch, batch_idx):\n        x = batch\n        loss = self.forward(x)\n        if self.global_step % self.log_steps == 0:\n            self.log('train/loss', loss['loss'])\n            self.log('train/loss_kl', loss['kl'], prog_bar=False)\n            self.log('train/loss_recon', loss['recon_loss'], prog_bar=False)\n            self._log_norm()\n\n        return loss\n    \n\n    def validation_step(self, batch, batch_idx):\n        x = batch\n        loss = self.forward(x)\n        self.log('val/loss_epoch', loss['loss'], on_epoch=True)\n        self.log('val_kl', loss['kl'], self.global_step)\n        self.log('val_recon_loss', loss['recon_loss'], self.global_step)\n\n        return loss\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.lr)\n\n\n    def _log_norm(self):\n        norm1 = sum(p.norm(1) for p in self.parameters())\n        norm1_grad = sum(p.grad.norm(1) for p in self.parameters() if p.grad is not None)\n        self.log('norm1_params', norm1)\n        self.log('norm1_grad', norm1_grad)\n\nclass VAEAnomalyTabular(VAEAnomalyDetection):\n\n    def make_encoder(self, input_size, latent_size):\n        \"\"\"\n        Simple encoder for tabular data.\n        If you want to feed image to a VAE make another encoder function with Conv2d instead of Linear layers.\n        :param input_size: number of input variables\n        :param latent_size: number of output variables i.e. the size of the latent space since it's the encoder of a VAE\n        :return: The untrained encoder model\n        \"\"\"\n        return nn.Sequential(\n            nn.Linear(input_size, 500),\n            nn.ReLU(),\n            nn.Linear(500, 200),\n            nn.ReLU(),\n            nn.Linear(200, latent_size * 2)\n            # times 2 because this is the concatenated vector of latent mean and variance\n        )\n\n    def make_decoder(self, latent_size, output_size):\n        \"\"\"\n        Simple decoder for tabular data.\n        :param latent_size: size of input latent space\n        :param output_size: number of output parameters. Must have the same value of input_size\n        :return: the untrained decoder\n        \"\"\"\n        return nn.Sequential(\n            nn.Linear(latent_size, 200),\n            nn.ReLU(),\n            nn.Linear(200, 500),\n            nn.ReLU(),\n            nn.Linear(500, output_size * 2)  # times 2 because this is the concatenated vector of reconstructed mean and variance\n        )\n\n\n"
  },
  {
    "path": "model/VAE_tf1.py",
    "content": "# ========== Legacy code ===============\n\n\nfrom math import ceil\n\nimport numpy as np\nimport tensorflow as tf\nfrom scipy.stats import multivariate_normal\n\n\ndef tf_namespace(namespace):\n    def wrapper(f):\n        def wrapped_f(*args, **kwargs):\n            with tf.name_scope(namespace):\n                return f(*args, **kwargs)\n\n        return wrapped_f\n\n    return wrapper\n\n\nclass VAE:\n\n    def __init__(self, input_shape, encode_sizes, latent_size, decode_sizes=None, mu_prior=None, sigma_prior=None,\n                 lr=10e-4,  momentum=0.9, save_model=True):\n        self.encode_sizes = encode_sizes\n        self.latent_size = latent_size\n        self.decode_sizes = decode_sizes or encode_sizes[::-1]\n        self.mu_prior = mu_prior or np.zeros([latent_size], dtype='float32')\n        self.sigma_prior = sigma_prior or np.ones([latent_size], 'float32')\n        self.lr = lr\n        self.momentum = momentum\n        self.input_shape = input_shape\n        self.save_model = save_model\n        self._build_graph(input_shape, latent_size)\n\n    def _build_graph(self, input_shape, latent_size):\n        self.graph = tf.Graph()\n        with self.graph.as_default():\n            self._create_placeholders(input_shape)\n            self._create_encoder(self.X)\n            self._create_latent_distribution(self.encoder, latent_size)\n            self._create_decoder(self.z)\n            self.loss = - self.elbo(self.X, self.decoder, self.mu, self.log_sigma_square, self.sigma_square,\n                                    tf.constant(self.mu_prior), tf.constant(self.sigma_prior))\n            self.opt = tf.train.AdamOptimizer(self.lr, self.momentum)\n            self.opt_op = self.opt.minimize(self.loss)\n            self.session = tf.InteractiveSession(graph=self.graph)\n        writer = tf.summary.FileWriter(logdir='logdir', graph=self.graph)\n        writer.flush()\n\n    @property\n    def k_init(self):\n        return {'kernel_initializer': tf.glorot_uniform_initializer()}\n\n    def elbo(self, X_true, X_pred, mu, log_sigma, sigma, mu_prior, sigma_prior):\n        epsilon = tf.constant(0.000001)\n        self.mae = tf.losses.absolute_difference(X_true, X_pred, reduction=tf.losses.Reduction.NONE)\n        self.mae_sum = tf.reduce_sum(self.mae, axis=1)\n        log_sigma_prior = tf.log(sigma_prior + epsilon)\n        mu_diff = mu - mu_prior\n        self.kl = log_sigma_prior - log_sigma - 1 + (sigma + tf.multiply(mu_diff, mu_diff)) / sigma_prior\n        self.kl_sum = tf.reduce_sum(self.kl, axis=1)\n        return tf.reduce_mean(- self.mae_sum - self.kl_sum)\n\n    @tf_namespace('placeholders')\n    def _create_placeholders(self, input_shape):\n        self.X = tf.placeholder(tf.float32, shape=[None, *input_shape], name='X')\n\n    @tf_namespace('encoder')\n    def _create_encoder(self, X):\n        self.encode_layers = []\n        self.encoder = X\n        for i, lsize in enumerate(self.encode_sizes):\n            self.encoder = tf.layers.dense(self.encoder, lsize, **self.k_init,\n                                           activation=tf.nn.relu, name=f'encoder_{i + 1}')\n            self.encode_layers.append(self.encoder)\n            setattr(self, f'encoder_{i + 1}', self.encoder)\n\n    @tf_namespace('latent')\n    def _create_latent_distribution(self, encoder, latent_dim):\n        self.mu = tf.layers.dense(encoder, latent_dim, **self.k_init, name='mu')\n        self.log_sigma_square = tf.layers.dense(encoder, latent_dim,\n                                                **self.k_init, name='log_sigma_square')\n        self.sigma_square = tf.exp(self.log_sigma_square, 'sigma_square')\n        self.z = tf.add(self.mu, self.sigma_square * tf.random.normal(tf.shape(self.sigma_square)), 'z')\n\n    @tf_namespace('decoder')\n    def _create_decoder(self, z):\n        self.decoder = z\n        self.decode_layers = []\n        for i, lsize in enumerate(self.decode_sizes):\n            self.decoder = tf.layers.dense(self.decoder, lsize, **self.k_init,\n                                           activation=tf.nn.relu, name=f'decoder_{i + 1}')\n            setattr(self, f'decoder_{i + 1}', self.decoder)\n            self.decode_layers.append(self.decoder)\n            if i == len(self.decode_sizes) - 1:\n                self.mu_post = tf.layers.dense(self.decoder, self.input_shape[0], name='mu_posterior')\n                self.log_sigma_post = tf.layers.dense(self.decoder, self.input_shape[0])\n                self.sigma_post = tf.exp(self.log_sigma_post, 'sigma_square_posterior')\n                self.decoder = tf.add(self.mu_post,\n                                      self.sigma_post * tf.random.normal((self.input_shape[0],), name='eps_post'),\n                                      name='decoder_output')\n                setattr(self, f'decoder_{i + 2}', self.decoder)\n                self.decode_layers.append(self.decoder)\n        return self.decoder\n\n    @property\n    def layers(self):\n        return [(f'encoder_{i}', getattr(self, f'encoder_{i}')) for i in range(1, len(self.encode_layers) + 1)] + \\\n               [('mu', self.mu), ('sigma', self.log_sigma_square), ('z', self.z)] + \\\n               [(f'decoder_{i}', getattr(self, f'decoder_{i}')) for i in range(1, len(self.decode_layers) + 1)]\n\n    def fit(self, X, epochs, batch_size, print_every=50, save_every_epochs=5, verbose=True):\n        n_batch = ceil(X.shape[0] / batch_size)\n        if self.save_model:\n            saver = tf.train.Saver()\n        self.session.run(tf.global_variables_initializer())\n        for epoch in range(1, epochs + 1):\n            np.random.shuffle(X)\n            acc_loss = 0\n            counter = 0\n            for i in range(n_batch):\n                slice_batch = slice(i * batch_size, (i + 1) * batch_size) if i != n_batch - 1 else slice(\n                    i * batch_size,\n                    None)\n                X_batch = X[slice_batch, :]\n                batch_loss, _ = self.session.run([self.loss, self.opt_op], {self.X: X_batch})\n                acc_loss += batch_loss\n                if verbose and counter % print_every == 0:\n                    print(f\" Epoch {epoch} - batch {i} - neg_ELBO = {batch_loss}\")\n                counter += 1\n            if verbose:\n                print(f'\\nEpoch {epoch} - Avg loss = {acc_loss / n_batch}')\n                print('\\n' + ('-' * 70))\n            if self.save_model and (epoch+1) % save_every_epochs == 0:\n                saver.save(self.session, \"ckpts/ad_vae.ckpt\")\n\n    def generate(self, n=1, mu_prior=None, sigma_prior=None):\n        \"\"\"\n        Generate new examples sampling from the latent distribution\n        :param n: number of examples to generate\n        :param mu_prior:\n        :param sigma_prior:\n        :return: a matrix of size [n, p] where p is the number of variables of X_train\n        \"\"\"\n        if mu_prior is None:\n            mu_prior = self.mu_prior\n        if sigma_prior is None:\n            sigma_prior = self.sigma_prior\n        z = np.random.multivariate_normal(mu_prior, np.diag(sigma_prior), [n])\n        return self.session.run(self.decoder, feed_dict={self.z: z})\n\n    def reconstruct(self, X):\n        return self.session.run(self.decoder, feed_dict={self.X: X})\n\n    def reconstructed_probability(self, X, L=100):\n        reconstructed_prob = np.zeros((X.shape[0],), dtype='float32')\n        mu_hat, sigma_hat = self.session.run([self.mu_post, self.sigma_post], {self.X: X})\n        for l in range(L):\n            mu_hat = mu_hat.reshape(X.shape)\n            sigma_hat = sigma_hat.reshape(X.shape) + 0.00001\n            for i in range(X.shape[0]):\n                p_l = multivariate_normal.pdf(X[i, :], mu_hat[i, :], np.diag(sigma_hat[i, :]))\n                reconstructed_prob[i] += p_l\n        reconstructed_prob /= L\n        return reconstructed_prob\n\n    def is_outlier(self, X, L=100, alpha=0.05):\n        p_hat = self.reconstructed_probability(X, L)\n        return p_hat < alpha\n\n    def open(self):\n        if not hasattr(self, 'session') or self.session is None:\n            if self.graph is None:\n                self._build_graph(self.input_shape, self.latent_size)\n            else:\n                self.session = tf.InteractiveSession(graph=self.graph)\n\n    def close(self):\n        if hasattr(VAE, 'session') and VAE.session is not None:\n            VAE.session.close()\n            VAE.session = None\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.close()\n\n    def __delete__(self, instance):\n        self.close()\n\n    def __setattr__(self, key, value):\n        if key == 'session':\n            if hasattr(self, 'session') and self.session is not None:\n                self.close()\n            VAE.session = value\n        else:\n            self.__dict__[key] = value\n\n    def __delattr__(self, item):\n        if item == 'session':\n            self.close()\n            del VAE.__dict__['session']\n        else:\n            del self.__dict__[item]\n\n    def __enter__(self):\n        self.open()"
  },
  {
    "path": "model/__init__.py",
    "content": "from .VAE import VAEAnomalyDetection, VAEAnomalyTabular"
  },
  {
    "path": "model/encoder_decoder.py",
    "content": "\"\"\"\nThis module contains simple encoder and decoder for tabular data.\nFor your own data you need to create your own encoder and decoder.\nHowever the input and output of your encoder and decoder must be the same of the ones in this module.\n\"\"\"\n\nfrom torch import nn\n\ndef tabular_encoder(input_size: int, latent_size: int):\n    \"\"\"\n    Simple encoder for tabular data.\n    If you want to feed image to a VAE make another encoder function with Conv2d instead of Linear layers.\n    \n    Parameters\n    ----------\n    input_size : int\n        number of input variables. In case of tabular data it's the number of columns.\n    latent_size : int\n        number of output variables i.e. the size of the latent space since it's the encoder of a VAE\n\n    Returns\n    -------\n    The untrained encoder model\n    \n    \"\"\"\n    return nn.Sequential(\n        nn.Linear(input_size, 500),\n        nn.ReLU(),\n        nn.Linear(500, 200),\n        nn.ReLU(),\n        nn.Linear(200, latent_size * 2)  # times 2 because this is the concatenated vector of latent mean and variance\n    )\n\n\ndef tabular_decoder(latent_size: int, output_size: int):\n    \"\"\"\n    Simple decoder for tabular data.\n\n    Parameters\n    ----------\n    latent_size : int\n        size of input latent space\n    output_size : int\n        number of output parameters. Must have the same value of input_size of the encoder\n\n    Returns\n    -------\n    The untrained decoder\n    \"\"\"\n    return nn.Sequential(\n        nn.Linear(latent_size, 200),\n        nn.ReLU(),\n        nn.Linear(200, 500),\n        nn.ReLU(),\n        nn.Linear(500, output_size * 2)\n        # times 2 because this is the concatenated vector of reconstructed mean and variance\n    )\n\n"
  },
  {
    "path": "pyproject.toml",
    "content": "[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n\n[project]\nname = \"vae_anomaly_detection\"\nversion = \"2.0.1\"\nrequires-python = \">3.6,<3.12\"\ndescription = \"Pytorch/TF1 implementation of Variational AutoEncoder for anomaly detection following the paper \\\"Variational Autoencoder based Anomaly Detection using Reconstruction Probability by Jinwon An, Sungzoon Cho\\\"\"\nauthors = [{name=\"Michele De Vita\", email=\"mik3dev@gmail.com\"}]\n\nclassifiers = [\n    \"Development Status :: 5 - Production/Stable\",\n    \"Intended Audience :: Developers\",\n    \"Intended Audience :: Science/Research\",\n    \"License :: OSI Approved :: MIT License\",\n    \"Operating System :: OS Independent\",\n    \"Programming Language :: Python\",\n    \"Programming Language :: Python :: 3\",\n    \"Programming Language :: Python :: 3.7\",\n    \"Programming Language :: Python :: 3.8\",\n    \"Programming Language :: Python :: 3.9\",\n    \"Programming Language :: Python :: 3.10\",\n    \"Programming Language :: Python :: 3.11\",\n    \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n    \"Topic :: Software Development :: Libraries :: Python Modules\"\n    ]\nkeywords = [\"vae\", \"anomaly detection\", \"deep learning\", \"pytorch\"]\nlicense = {text = \"MIT\"}\nreadme = \"readme.md\"\n\ndependencies = [\n\"path>=15.0\",\n\"torch>=1.8\",\n\"pytorch-lightning>=1.9\",\n\"PyYAML>=5.0\",\n\"tqdm>=4.0\",\n\"tensorboard>=0.20\",\n\"numpy>= 1.18\",\n]\n\n[project.urls]\nhomepage = \"https://github.com/Michedev/VAE_anomaly_detection\"\nrepository = \"https://github.com/Michedev/VAE_anomaly_detection\"\n\n[project.optional-dependencies]\ndev = [\n    \"pytest\",\n]\n\n[tool.hatch.envs.default]\npython = \"3.10\"\ndependencies = [\n    \"torch>=1.8\",\n    \"pytorch-lightning\",\n    \"path\",\n    \"tensorboard\",\n    \"numpy\",\n    \"torchvision\",\n]\n\n[tool.hatch.envs.default.scripts]\ntrain = \"python train.py -i 100 -l 32 {args:train}\"\n\n\n[tool.hatch.envs.cpu]\npython = \"3.10\"\ndependencies = [\n    \"torch>=1.8\",\n    \"pytorch-lightning\",\n    \"path\",\n    \"tensorboard\",\n    \"numpy\",\n    \"torchvision\",\n]\n\n[tool.hatch.envs.cpu.env-vars]\nPIP_EXTRA_INDEX_URL = \"https://download.pytorch.org/whl/cpu\"\n\n\n[tool.hatch.envs.test]\npython = \"python3\"\ndependencies = [\n    \"torch>=1.8\",\n    \"pytorch-lightning\",\n    \"path\",\n    \"tensorboard\",\n    \"numpy\",\n    \"torchvision\",\n    \"pytest\",\n]\n\n[tool.hatch.envs.test.overrides]\nmatrix.foo.set-python = [\"3.6\", \"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.11\"]\n\n\n[tool.hatch.envs.test.env-vars]\nPIP_EXTRA_INDEX_URL = \"https://download.pytorch.org/whl/cpu\"\n\n\n[tool.hatch.build]\ninclude = [\"vae_anomaly_detection\"]"
  },
  {
    "path": "readme.md",
    "content": "# Variational autoencoder for anomaly detection\n\n![PyPI](https://img.shields.io/pypi/v/vae-anomaly-detection?style=flat-square)\n![PyPI - Python Version](https://img.shields.io/pypi/pyversions/vae-anomaly-detection?style=flat-square)\n![PyPI - License](https://img.shields.io/pypi/l/vae-anomaly-detection?style=flat-square)\n![PyPI - Downloads](https://img.shields.io/pypi/dm/vae-anomaly-detection?style=flat-square)\n\nPytorch/TF1 implementation of Variational AutoEncoder for anomaly detection following the paper\n [Variational Autoencoder based Anomaly Detection using Reconstruction Probability by Jinwon An, Sungzoon Cho](https://www.semanticscholar.org/paper/Variational-Autoencoder-based-Anomaly-Detection-An-Cho/061146b1d7938d7a8dae70e3531a00fceb3c78e8)\n <br>\n\n## How to install\n\n#### Python package way\n _pip_ package containing the model and training_step only \n   \n    pip install vae-anomaly-detection\n\n\n#### Hack this repository\n\n\n   a. Clone the repo\n\n    git clone git@github.com:Michedev/VAE_anomaly_detection.git\n\n   b. Install hatch\n\n    pip install hatch\n\n   c. Make the environment with torch gpu support\n\n    hatch env create\n      \nor with cpu support\n\n    hatch env create cpu\n\n   d. Run the train\n\n    hatch run train\n\nor in cpu\n          \n    hatch run cpu:train\n\n   To know all the train parameters run `hatch run train --help`\n\n\n\n\nThis version contains the model and the training procedure\n\n## How To Train your Model\n\n- Define your dataset into dataset.py and overwrite the line `train_set = rand_dataset()  # set here your dataset` in `train.py`\n- Subclass VAEAnomalyDetection and define the methods `make_encoder` and `make_decoder`. The output of `make_encoder` should be a flat vector while the output of `make_decoder should have the same shape of the input.\n## Make your model\n\nSubclass ```VAEAnomalyDetection``` and define your encoder and decoder like in ```VaeAnomalyTabular```\n\n```python\nclass VAEAnomalyTabular(VAEAnomalyDetection):\n\n    def make_encoder(self, input_size, latent_size):\n        \"\"\"\n        Simple encoder for tabular data.\n        If you want to feed image to a VAE make another encoder function with Conv2d instead of Linear layers.\n        :param input_size: number of input variables\n        :param latent_size: number of output variables i.e. the size of the latent space since it's the encoder of a VAE\n        :return: The untrained encoder model\n        \"\"\"\n        return nn.Sequential(\n            nn.Linear(input_size, 500),\n            nn.ReLU(),\n            nn.Linear(500, 200),\n            nn.ReLU(),\n            nn.Linear(200, latent_size * 2)\n            # times 2 because this is the concatenated vector of latent mean and variance\n        )\n\n    def make_decoder(self, latent_size, output_size):\n        \"\"\"\n        Simple decoder for tabular data.\n        :param latent_size: size of input latent space\n        :param output_size: number of output parameters. Must have the same value of input_size\n        :return: the untrained decoder\n        \"\"\"\n        return nn.Sequential(\n            nn.Linear(latent_size, 200),\n            nn.ReLU(),\n            nn.Linear(200, 500),\n            nn.ReLU(),\n            nn.Linear(500, output_size * 2)  # times 2 because this is the concatenated vector of reconstructed mean and variance\n        )\n```\n\n## How to make predictions:\nOnce the model is trained (suppose for simplicity that it is under _saved_models/{train-datetime}/_ ) just load and predict with this code snippet:\n```python\nimport torch\n\n#load X_test\nmodel = VaeAnomalyTabular.load_checkpoint('saved_models/2022-01-06_15-12-23/last.ckpt')\n# load saved parameters from a run\noutliers = model.is_anomaly(X_test)\n```\n\n\n## train.py help\n\n        usage: train.py [-h] --input-size INPUT_SIZE --latent-size LATENT_SIZE\n                        [--num-resamples NUM_RESAMPLES] [--epochs EPOCHS] [--batch-size BATCH_SIZE]\n                        [--device {cpu,gpu,tpu}] [--lr LR] [--no-progress-bar]\n                        [--steps-log-loss STEPS_LOG_LOSS]\n                        [--steps-log-norm-params STEPS_LOG_NORM_PARAMS]\n\n        options:\n        -h, --help            show this help message and exit\n        --input-size INPUT_SIZE, -i INPUT_SIZE\n                                Number of input features. In 1D case it is the vector length, in 2D\n                                case it is the number of channels\n        --latent-size LATENT_SIZE, -l LATENT_SIZE\n                                Size of the latent space\n        --num-resamples NUM_RESAMPLES, -L NUM_RESAMPLES\n                                Number of resamples in the latent distribution during training\n        --epochs EPOCHS, -e EPOCHS\n                                Number of epochs to train for\n        --batch-size BATCH_SIZE, -b BATCH_SIZE\n        --device {cpu,gpu,tpu}, -d {cpu,gpu,tpu}, --accelerator {cpu,gpu,tpu}\n                                Device to use for training. Can be cpu, gpu or tpu\n        --lr LR               Learning rate\n        --no-progress-bar\n        --steps-log-loss STEPS_LOG_LOSS\n                                Number of steps between each loss logging\n        --steps-log-norm-params STEPS_LOG_NORM_PARAMS\n                                Number of steps between each model parameters logging\n"
  },
  {
    "path": "tests/__init__.py",
    "content": ""
  },
  {
    "path": "tests/test_pytorch_model.py",
    "content": "import torch\nfrom model import VAEAnomalyTabular\n\n\ndef test_pytorch_anomaly_detection():\n    batch_size = 32\n    input_size = 100\n    latent_size = 32\n    model = VAEAnomalyTabular(input_size, latent_size, L=2)\n    batch = torch.rand(batch_size, input_size)\n    batch_anomaly = model.is_anomaly(batch, alpha=0.05)\n    assert batch_anomaly.shape == (batch_size,)\n    assert batch_anomaly.dtype == torch.bool\n\n\ndef test_pytorch_prediction():\n    batch_size = 32\n    input_size = 100\n    latent_size = 32\n    model = VAEAnomalyTabular(input_size, latent_size, L=2)\n    batch = torch.rand(batch_size, input_size)\n    reconstructed_probability = model.reconstructed_probability(batch)\n    assert reconstructed_probability.shape == (batch_size,)\n    assert reconstructed_probability.dtype == torch.float\n    assert 1.0 >= reconstructed_probability.max().item() and \\\n           reconstructed_probability.min().item() >= 0.0\n    \n\ndef test_training_step():\n    batch_size = 32\n    input_size = 100\n    latent_size = 32\n    model = VAEAnomalyTabular(input_size, latent_size, L=2)\n    batch = torch.rand(batch_size, input_size)\n    loss_dict = model.training_step(batch, batch_idx=0)\n    loss = loss_dict['loss']\n    assert loss.numel() == 1\n    assert loss.dtype == torch.float    \n"
  },
  {
    "path": "train.py",
    "content": "import argparse\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nimport torch\nimport yaml\nfrom path import Path\nfrom torch.utils.data import DataLoader\nfrom torch.utils.tensorboard import SummaryWriter\nfrom datetime import datetime\n\nfrom model.VAE import VAEAnomalyTabular\nfrom dataset import rand_dataset\n\nROOT = Path(__file__).parent\nSAVED_MODELS = ROOT / 'saved_models'\n\n\ndef make_folder_run() -> Path:\n    \"\"\"\n    Get the folder where to store the experiment. \n    The folder is named with the current date and time.\n    \n    Returns:\n        Path: the path to the folder where to store the experiment\n    \"\"\"\n    checkpoint_folder = SAVED_MODELS / datetime.now().strftime('%Y-%m-%d_%H-%M-%S')\n    checkpoint_folder.makedirs_p()\n    return checkpoint_folder\n\n\ndef get_args() -> argparse.Namespace:\n    \"\"\"\n    Parse command line arguments\n    \n    Returns:\n        argparse.Namespace: the parsed arguments\n    \"\"\"\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--input-size', '-i', type=int, required=True, dest='input_size', help='Number of input features. In 1D case it is the vector length, in 2D case it is the number of channels')\n    parser.add_argument('--latent-size', '-l', type=int, required=True, dest='latent_size', help='Size of the latent space')\n    parser.add_argument('--num-resamples', '-L', type=int, dest='num_resamples', default=10,\n                        help='Number of resamples in the latent distribution during training')\n    parser.add_argument('--epochs', '-e', type=int, dest='epochs', default=100, help='Number of epochs to train for')\n    parser.add_argument('--batch-size', '-b', type=int, dest='batch_size', default=32)\n    parser.add_argument('--device', '-d', '--accelerator', type=str, dest='device', default='gpu', help='Device to use for training. Can be cpu, gpu or tpu', choices=['cpu', 'gpu', 'tpu'])\n    parser.add_argument('--lr', type=float, dest='lr', default=1e-3, help='Learning rate')\n    parser.add_argument('--no-progress-bar', action='store_true', dest='no_progress_bar')\n    parser.add_argument('--steps-log-loss', type=int, dest='steps_log_loss', default=1_000, help='Number of steps between each loss logging')\n    parser.add_argument('--steps-log-norm-params', type=int, \n                        dest='steps_log_norm_params', default=1_000, help='Number of steps between each model parameters logging')\n\n    return parser.parse_args()\n\n\ndef main():\n    \"\"\"\n    Main function to train the VAE model\n    \"\"\"\n    args = get_args()\n    print(args)\n    experiment_folder = make_folder_run()\n\n    # copy model folder into experiment folder\n    ROOT.joinpath('model').copytree(experiment_folder / 'model')\n\n    with open(experiment_folder / 'config.yaml', 'w') as f:\n        yaml.dump(args, f)\n\n    model = VAEAnomalyTabular(args.input_size, args.latent_size, args.num_resamples, lr=args.lr)\n\n    train_set = rand_dataset()  # set here your dataset\n    train_dloader = DataLoader(train_set, args.batch_size)\n\n    val_dataset = rand_dataset()  # set here your dataset\n    val_dloader = DataLoader(val_dataset, args.batch_size)\n\n    checkpoint = ModelCheckpoint(\n        filepath=experiment_folder / '{epoch:02d}-{val_loss:.2f}',\n        save_top_k=1,\n        verbose=True,\n        monitor='val_loss',\n        mode='min',\n        prefix='',\n        save_last=True,\n    )\n\n    trainer = Trainer(callbacks=[checkpoint],)\n    trainer.fit(model, train_dloader, val_dloader)\n\n\nif __name__ == '__main__':\n    main()"
  }
]