[ { "path": ".gitignore", "content": "# Custom\r\n*.png\r\n*.h5\r\n*.npz\r\n*.csv\r\n*.zip\r\n\r\n# Byte-compiled / optimized / DLL files\r\n__pycache__/\r\n*.py[cod]\r\n*$py.class\r\n\r\n# C extensions\r\n*.so\r\n\r\n# Distribution / packaging\r\n.Python\r\nenv/\r\nbuild/\r\ndevelop-eggs/\r\ndist/\r\ndownloads/\r\neggs/\r\n.eggs/\r\nlib/\r\nlib64/\r\nparts/\r\nsdist/\r\nvar/\r\n*.egg-info/\r\n.installed.cfg\r\n*.egg\r\n\r\n# PyInstaller\r\n# Usually these files are written by a python script from a template\r\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\r\n*.manifest\r\n*.spec\r\n\r\n# Installer logs\r\npip-log.txt\r\npip-delete-this-directory.txt\r\n\r\n# Unit test / coverage reports\r\nhtmlcov/\r\n.tox/\r\n.coverage\r\n.coverage.*\r\n.cache\r\nnosetests.xml\r\ncoverage.xml\r\n*,cover\r\n.hypothesis/\r\n\r\n# Translations\r\n*.mo\r\n*.pot\r\n\r\n# Django stuff:\r\n*.log\r\nlocal_settings.py\r\n\r\n# Flask stuff:\r\ninstance/\r\n.webassets-cache\r\n\r\n# Scrapy stuff:\r\n.scrapy\r\n\r\n# Sphinx documentation\r\ndocs/_build/\r\n\r\n# PyBuilder\r\ntarget/\r\n\r\n# IPython Notebook\r\n.ipynb_checkpoints\r\n\r\n# pyenv\r\n.python-version\r\n\r\n# celery beat schedule file\r\ncelerybeat-schedule\r\n\r\n# dotenv\r\n.env\r\n\r\n# virtualenv\r\nvenv/\r\nENV/\r\n\r\n# Spyder project settings\r\n.spyderproject\r\n\r\n# Rope project settings\r\n.ropeproject\r\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2018 izikgo\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# Deep Anomaly Detection Using Geometric Transformations\nTo be presented in NIPS 2018 by Izhak Golan and Ran El-Yaniv.\n\n## Introduction\nThis is the official implementation of \"Deep Anomaly Detection Using Geometric Transformations\".\nIt includes all experiments reported in the paper.\n\n## Requirements\n* Python 3.5+\n* Keras 2.2.0\n* Tensorflow 1.8.0\n* sklearn 0.19.1\n\n## Citation\nIf you use the ideas or method presented in the paper, please cite:\n\n```\n@inproceedings{NEURIPS2018_5e62d03a,\n author = {Golan, Izhak and El-Yaniv, Ran},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},\n pages = {},\n publisher = {Curran Associates, Inc.},\n title = {Deep Anomaly Detection Using Geometric Transformations},\n url = {https://proceedings.neurips.cc/paper/2018/file/5e62d03aec0d17facfc5355dd90d441c-Paper.pdf},\n volume = {31},\n year = {2018}\n}\n```\n" }, { "path": "experiments.py", "content": "import os\r\nimport csv\r\nfrom collections import defaultdict\r\nfrom glob import glob\r\nfrom datetime import datetime\r\nfrom multiprocessing import Manager, freeze_support, Process\r\nimport numpy as np\r\nimport scipy.stats\r\nfrom scipy.special import psi, polygamma\r\nfrom sklearn.metrics import roc_auc_score\r\nfrom sklearn.svm import OneClassSVM\r\nfrom sklearn.model_selection import ParameterGrid\r\nfrom sklearn.externals.joblib import Parallel, delayed\r\nfrom keras.models import Model, Input, Sequential\r\nfrom keras.layers import Dense, Dropout\r\nfrom keras.utils import to_categorical\r\nfrom utils import load_cifar10, load_cats_vs_dogs, load_fashion_mnist, load_cifar100\r\nfrom utils import save_roc_pr_curve_data, get_class_name_from_index, get_channels_axis\r\nfrom transformations import Transformer\r\nfrom models.wide_residual_network import create_wide_residual_network\r\nfrom models.encoders_decoders import conv_encoder, conv_decoder\r\nfrom models import dsebm, dagmm, adgan\r\nimport keras.backend as K\r\n\r\nRESULTS_DIR = ''\r\n\r\n\r\ndef _transformations_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):\r\n gpu_to_use = gpu_q.get()\r\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_to_use\r\n\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n\r\n if dataset_name in ['cats-vs-dogs']:\r\n transformer = Transformer(16, 16)\r\n n, k = (16, 8)\r\n else:\r\n transformer = Transformer(8, 8)\r\n n, k = (10, 4)\r\n mdl = create_wide_residual_network(x_train.shape[1:], transformer.n_transforms, n, k)\r\n mdl.compile('adam',\r\n 'categorical_crossentropy',\r\n ['acc'])\r\n\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task))\r\n x_train_task_transformed = transformer.transform_batch(np.repeat(x_train_task, transformer.n_transforms, axis=0),\r\n transformations_inds)\r\n batch_size = 128\r\n\r\n mdl.fit(x=x_train_task_transformed, y=to_categorical(transformations_inds),\r\n batch_size=batch_size, epochs=int(np.ceil(200/transformer.n_transforms))\r\n )\r\n\r\n #################################################################################################\r\n # simplified normality score\r\n #################################################################################################\r\n # preds = np.zeros((len(x_test), transformer.n_transforms))\r\n # for t in range(transformer.n_transforms):\r\n # preds[:, t] = mdl.predict(transformer.transform_batch(x_test, [t] * len(x_test)),\r\n # batch_size=batch_size)[:, t]\r\n #\r\n # labels = y_test.flatten() == single_class_ind\r\n # scores = preds.mean(axis=-1)\r\n #################################################################################################\r\n\r\n def calc_approx_alpha_sum(observations):\r\n N = len(observations)\r\n f = np.mean(observations, axis=0)\r\n\r\n return (N * (len(f) - 1) * (-psi(1))) / (\r\n N * np.sum(f * np.log(f)) - np.sum(f * np.sum(np.log(observations), axis=0)))\r\n\r\n def inv_psi(y, iters=5):\r\n # initial estimate\r\n cond = y >= -2.22\r\n x = cond * (np.exp(y) + 0.5) + (1 - cond) * -1 / (y - psi(1))\r\n\r\n for _ in range(iters):\r\n x = x - (psi(x) - y) / polygamma(1, x)\r\n return x\r\n\r\n def fixed_point_dirichlet_mle(alpha_init, log_p_hat, max_iter=1000):\r\n alpha_new = alpha_old = alpha_init\r\n for _ in range(max_iter):\r\n alpha_new = inv_psi(psi(np.sum(alpha_old)) + log_p_hat)\r\n if np.sqrt(np.sum((alpha_old - alpha_new) ** 2)) < 1e-9:\r\n break\r\n alpha_old = alpha_new\r\n return alpha_new\r\n\r\n def dirichlet_normality_score(alpha, p):\r\n return np.sum((alpha - 1) * np.log(p), axis=-1)\r\n\r\n scores = np.zeros((len(x_test),))\r\n observed_data = x_train_task\r\n for t_ind in range(transformer.n_transforms):\r\n observed_dirichlet = mdl.predict(transformer.transform_batch(observed_data, [t_ind] * len(observed_data)),\r\n batch_size=1024)\r\n log_p_hat_train = np.log(observed_dirichlet).mean(axis=0)\r\n\r\n alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet)\r\n alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx\r\n\r\n mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train)\r\n\r\n x_test_p = mdl.predict(transformer.transform_batch(x_test, [t_ind] * len(x_test)),\r\n batch_size=1024)\r\n scores += dirichlet_normality_score(mle_alpha_t, x_test_p)\r\n\r\n scores /= transformer.n_transforms\r\n labels = y_test.flatten() == single_class_ind\r\n\r\n res_file_name = '{}_transformations_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n mdl_weights_name = '{}_transformations_{}_{}_weights.h5'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n mdl_weights_path = os.path.join(RESULTS_DIR, dataset_name, mdl_weights_name)\r\n mdl.save_weights(mdl_weights_path)\r\n\r\n gpu_q.put(gpu_to_use)\r\n\r\n\r\ndef _train_ocsvm_and_score(params, xtrain, test_labels, xtest):\r\n return roc_auc_score(test_labels, OneClassSVM(**params).fit(xtrain).decision_function(xtest))\r\n\r\n\r\ndef _raw_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind):\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n\r\n x_train = x_train.reshape((len(x_train), -1))\r\n x_test = x_test.reshape((len(x_test), -1))\r\n\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n if dataset_name in ['cats-vs-dogs']: # OC-SVM is quadratic on the number of examples, so subsample training set\r\n subsample_inds = np.random.choice(len(x_train_task), 5000, replace=False)\r\n x_train_task = x_train_task[subsample_inds]\r\n\r\n pg = ParameterGrid({'nu': np.linspace(0.1, 0.9, num=9),\r\n 'gamma': np.logspace(-7, 2, num=10, base=2)})\r\n\r\n results = Parallel(n_jobs=6)(\r\n delayed(_train_ocsvm_and_score)(d, x_train_task, y_test.flatten() == single_class_ind, x_test)\r\n for d in pg)\r\n\r\n best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])\r\n best_ocsvm = OneClassSVM(**best_params).fit(x_train_task)\r\n scores = best_ocsvm.decision_function(x_test)\r\n labels = y_test.flatten() == single_class_ind\r\n\r\n res_file_name = '{}_raw-oc-svm_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n\r\ndef _cae_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):\r\n gpu_to_use = gpu_q.get()\r\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_to_use\r\n\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n\r\n n_channels = x_train.shape[get_channels_axis()]\r\n input_side = x_train.shape[2] # channel side will always be at shape[2]\r\n enc = conv_encoder(input_side, n_channels)\r\n dec = conv_decoder(input_side, n_channels)\r\n x_in = Input(shape=x_train.shape[1:])\r\n x_rec = dec(enc(x_in))\r\n cae = Model(x_in, x_rec)\r\n cae.compile('adam', 'mse')\r\n\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring\r\n cae.fit(x=x_train_task, y=x_train_task, batch_size=128, epochs=200, validation_data=(x_test_task, x_test_task))\r\n\r\n x_train_task_rep = enc.predict(x_train_task, batch_size=128)\r\n if dataset_name in ['cats-vs-dogs']: # OC-SVM is quadratic on the number of examples, so subsample training set\r\n subsample_inds = np.random.choice(len(x_train_task_rep), 2500, replace=False)\r\n x_train_task_rep = x_train_task_rep[subsample_inds]\r\n\r\n x_test_rep = enc.predict(x_test, batch_size=128)\r\n pg = ParameterGrid({'nu': np.linspace(0.1, 0.9, num=9),\r\n 'gamma': np.logspace(-7, 2, num=10, base=2)})\r\n\r\n results = Parallel(n_jobs=6)(\r\n delayed(_train_ocsvm_and_score)(d, x_train_task_rep, y_test.flatten() == single_class_ind, x_test_rep)\r\n for d in pg)\r\n\r\n best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])\r\n print(best_params)\r\n best_ocsvm = OneClassSVM(**best_params).fit(x_train_task_rep)\r\n scores = best_ocsvm.decision_function(x_test_rep)\r\n labels = y_test.flatten() == single_class_ind\r\n\r\n res_file_name = '{}_cae-oc-svm_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n gpu_q.put(gpu_to_use)\r\n\r\n\r\ndef _dsebm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):\r\n gpu_to_use = gpu_q.get()\r\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_to_use\r\n\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n\r\n n_channels = x_train.shape[get_channels_axis()]\r\n input_side = x_train.shape[2] # image side will always be at shape[2]\r\n encoder_mdl = conv_encoder(input_side, n_channels, representation_activation='relu')\r\n energy_mdl = dsebm.create_energy_model(encoder_mdl)\r\n reconstruction_mdl = dsebm.create_reconstruction_model(energy_mdl)\r\n\r\n # optimization parameters\r\n batch_size = 128\r\n epochs = 200\r\n reconstruction_mdl.compile('adam', 'mse')\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring\r\n reconstruction_mdl.fit(x=x_train_task, y=x_train_task,\r\n batch_size=batch_size,\r\n epochs=epochs,\r\n validation_data=(x_test_task, x_test_task))\r\n\r\n scores = -energy_mdl.predict(x_test, batch_size)\r\n labels = y_test.flatten() == single_class_ind\r\n res_file_name = '{}_dsebm_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n gpu_q.put(gpu_to_use)\r\n\r\n\r\ndef _dagmm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):\r\n gpu_to_use = gpu_q.get()\r\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_to_use\r\n\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n\r\n n_channels = x_train.shape[get_channels_axis()]\r\n input_side = x_train.shape[2] # image side will always be at shape[2]\r\n enc = conv_encoder(input_side, n_channels, representation_dim=5,\r\n representation_activation='linear')\r\n dec = conv_decoder(input_side, n_channels=n_channels, representation_dim=enc.output_shape[-1])\r\n n_components = 3\r\n estimation = Sequential([Dense(64, activation='tanh', input_dim=enc.output_shape[-1] + 2), Dropout(0.5),\r\n Dense(10, activation='tanh'), Dropout(0.5),\r\n Dense(n_components, activation='softmax')]\r\n )\r\n\r\n batch_size = 256\r\n epochs = 200\r\n lambda_diag = 0.0005\r\n lambda_energy = 0.01\r\n dagmm_mdl = dagmm.create_dagmm_model(enc, dec, estimation, lambda_diag)\r\n dagmm_mdl.compile('adam', ['mse', lambda y_true, y_pred: lambda_energy*y_pred])\r\n\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring\r\n dagmm_mdl.fit(x=x_train_task, y=[x_train_task, np.zeros((len(x_train_task), 1))], # second y is dummy\r\n batch_size=batch_size,\r\n epochs=epochs,\r\n validation_data=(x_test_task, [x_test_task, np.zeros((len(x_test_task), 1))]),\r\n # verbose=0\r\n )\r\n\r\n energy_mdl = Model(dagmm_mdl.input, dagmm_mdl.output[-1])\r\n\r\n scores = -energy_mdl.predict(x_test, batch_size)\r\n scores = scores.flatten()\r\n if not np.all(np.isfinite(scores)):\r\n min_finite = np.min(scores[np.isfinite(scores)])\r\n scores[~np.isfinite(scores)] = min_finite - 1\r\n labels = y_test.flatten() == single_class_ind\r\n res_file_name = '{}_dagmm_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n gpu_q.put(gpu_to_use)\r\n\r\n\r\ndef _adgan_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):\r\n gpu_to_use = gpu_q.get()\r\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = gpu_to_use\r\n\r\n (x_train, y_train), (x_test, y_test) = dataset_load_fn()\r\n if len(x_test) > 5000:\r\n # subsample x_test due to runtime complexity\r\n chosen_inds = np.random.choice(len(x_test), 5000, replace=False)\r\n x_test = x_test[chosen_inds]\r\n y_test = y_test[chosen_inds]\r\n\r\n n_channels = x_train.shape[get_channels_axis()]\r\n input_side = x_train.shape[2] # image side will always be at shape[2]\r\n critic = conv_encoder(input_side, n_channels, representation_dim=1,\r\n representation_activation='linear')\r\n noise_size = 256\r\n generator = conv_decoder(input_side, n_channels=n_channels, representation_dim=noise_size)\r\n\r\n def prior_gen(b_size):\r\n return np.random.normal(size=(b_size, noise_size))\r\n\r\n batch_size = 128\r\n epochs = 100\r\n\r\n x_train_task = x_train[y_train.flatten() == single_class_ind]\r\n\r\n def data_gen(b_size):\r\n chosen_inds = np.random.choice(len(x_train_task), b_size, replace=False)\r\n return x_train_task[chosen_inds]\r\n\r\n adgan.train_wgan_with_grad_penalty(prior_gen, generator, data_gen, critic, batch_size, epochs, grad_pen_coef=20)\r\n\r\n scores = adgan.scores_from_adgan_generator(x_test, prior_gen, generator)\r\n labels = y_test.flatten() == single_class_ind\r\n res_file_name = '{}_adgan_{}_{}.npz'.format(dataset_name,\r\n get_class_name_from_index(single_class_ind, dataset_name),\r\n datetime.now().strftime('%Y-%m-%d-%H%M'))\r\n res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)\r\n save_roc_pr_curve_data(scores, labels, res_file_path)\r\n\r\n gpu_q.put(gpu_to_use)\r\n\r\n\r\ndef run_experiments(load_dataset_fn, dataset_name, q, n_classes):\r\n\r\n # CAE OC-SVM\r\n processes = [Process(target=_cae_ocsvm_experiment,\r\n args=(load_dataset_fn, dataset_name, c, q)) for c in range(n_classes)]\r\n for p in processes:\r\n p.start()\r\n p.join()\r\n\r\n # Raw OC-SVM\r\n for c in range(n_classes):\r\n _raw_ocsvm_experiment(load_dataset_fn, dataset_name, c)\r\n\r\n n_runs = 5\r\n\r\n # Transformations\r\n for _ in range(n_runs):\r\n processes = [Process(target=_transformations_experiment,\r\n args=(load_dataset_fn, dataset_name, c, q)) for c in range(n_classes)]\r\n if dataset_name in ['cats-vs-dogs']: # Self-labeled set is memory consuming\r\n for p in processes:\r\n p.start()\r\n p.join()\r\n else:\r\n for p in processes:\r\n p.start()\r\n for p in processes:\r\n p.join()\r\n\r\n # DSEBM\r\n for _ in range(n_runs):\r\n processes = [Process(target=_dsebm_experiment,\r\n args=(load_dataset_fn, dataset_name, c, q)) for c in range(n_classes)]\r\n for p in processes:\r\n p.start()\r\n for p in processes:\r\n p.join()\r\n\r\n # DAGMM\r\n for _ in range(n_runs):\r\n processes = [Process(target=_dagmm_experiment,\r\n args=(load_dataset_fn, dataset_name, c, q)) for c in range(n_classes)]\r\n for p in processes:\r\n p.start()\r\n for p in processes:\r\n p.join()\r\n\r\n # ADGAN\r\n processes = [Process(target=_adgan_experiment,\r\n args=(load_dataset_fn, dataset_name, c, q)) for c in range(n_classes)]\r\n for p in processes:\r\n p.start()\r\n for p in processes:\r\n p.join()\r\n\r\n\r\ndef create_auc_table(metric='roc_auc'):\r\n file_path = glob(os.path.join(RESULTS_DIR, '*', '*.npz'))\r\n results = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))\r\n methods = set()\r\n for p in file_path:\r\n _, f_name = os.path.split(p)\r\n dataset_name, method, single_class_name = f_name.split(sep='_')[:3]\r\n methods.add(method)\r\n npz = np.load(p)\r\n roc_auc = npz[metric]\r\n results[dataset_name][single_class_name][method].append(roc_auc)\r\n\r\n for ds_name in results:\r\n for sc_name in results[ds_name]:\r\n for method_name in results[ds_name][sc_name]:\r\n roc_aucs = results[ds_name][sc_name][method_name]\r\n results[ds_name][sc_name][method_name] = [np.mean(roc_aucs),\r\n 0 if len(roc_aucs) == 1 else scipy.stats.sem(np.array(roc_aucs))\r\n ]\r\n\r\n with open('results-{}.csv'.format(metric), 'w') as csvfile:\r\n fieldnames = ['dataset', 'single class name'] + sorted(list(methods))\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n\r\n writer.writeheader()\r\n for ds_name in sorted(results.keys()):\r\n for sc_name in sorted(results[ds_name].keys()):\r\n row_dict = {'dataset': ds_name, 'single class name': sc_name}\r\n row_dict.update({method_name: '{:.3f} ({:.3f})'.format(*results[ds_name][sc_name][method_name])\r\n for method_name in results[ds_name][sc_name]})\r\n writer.writerow(row_dict)\r\n\r\n\r\nif __name__ == '__main__':\r\n freeze_support()\r\n N_GPUS = 2\r\n man = Manager()\r\n q = man.Queue(N_GPUS)\r\n for g in range(N_GPUS):\r\n q.put(str(g))\r\n\r\n experiments_list = [\r\n (load_cifar10, 'cifar10', 10),\r\n (load_cifar100, 'cifar100', 20),\r\n (load_fashion_mnist, 'fashion-mnist', 10),\r\n (load_cats_vs_dogs, 'cats-vs-dogs', 2),\r\n ]\r\n\r\n for data_load_fn, dataset_name, n_classes in experiments_list:\r\n run_experiments(data_load_fn, dataset_name, q, n_classes)\r\n\r\n" }, { "path": "intuition_experiments.py", "content": "import itertools\r\nimport numpy as np\r\nfrom scipy.ndimage.filters import gaussian_filter\r\nimport matplotlib.pyplot as plt\r\nfrom transformations import SimpleTransformer\r\nfrom utils import load_mnist\r\nfrom keras.utils import to_categorical\r\nfrom keras.layers import Flatten, Conv2D, Dense, BatchNormalization, MaxPool2D, Input, Lambda, average\r\nfrom keras.models import Sequential, Model\r\nimport keras.backend as K\r\nimport tensorflow as tf\r\n\r\n\r\n(x_train, y_train), (x_test, y_test) = load_mnist()\r\n# scale to be in [0, 1]\r\nx_train = (x_train + 1) / 2.\r\nx_test = (x_test + 1) / 2.\r\n\r\nsingle_class_ind = 3\r\nanomaly_class_ind = 0\r\n\r\nx_train_single = x_train[y_train == single_class_ind]\r\nx_test_single = x_test[y_test == single_class_ind]\r\nx_test_anomaly = x_test[y_test == anomaly_class_ind]\r\n\r\ntransformer = SimpleTransformer()\r\ntransformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_single))\r\nx_train_single_transformed = transformer.transform_batch(np.repeat(x_train_single, transformer.n_transforms, axis=0),\r\n transformations_inds)\r\n\r\n\r\nmdl = Sequential([Conv2D(32, (3, 3), padding='same', input_shape=(32, 32, 1), activation='relu'),\r\n BatchNormalization(axis=-1),\r\n MaxPool2D(),\r\n Flatten(),\r\n Dense(10, activation='relu'),\r\n BatchNormalization(axis=-1),\r\n Dense(transformer.n_transforms, activation='softmax')])\r\n\r\nmdl.compile('adam',\r\n 'categorical_crossentropy',\r\n ['acc'])\r\n\r\nbatch_size = 64\r\nmdl.fit(x=x_train_single_transformed,\r\n y=to_categorical(transformations_inds),\r\n batch_size=batch_size,\r\n validation_split=0.1,\r\n epochs=10)\r\n\r\nsingle_class_preds = np.zeros((len(x_test_single), transformer.n_transforms))\r\nfor t in range(transformer.n_transforms):\r\n single_class_preds[:, t] = mdl.predict(transformer.transform_batch(x_test_single, [t] * len(x_test_single)),\r\n batch_size=batch_size)[:, t]\r\nsingle_class_scores = single_class_preds.mean(axis=-1)\r\n\r\nanomaly_class_preds = np.zeros((len(x_test_anomaly), transformer.n_transforms))\r\nfor t in range(transformer.n_transforms):\r\n anomaly_class_preds[:, t] = mdl.predict(transformer.transform_batch(x_test_anomaly, [t] * len(x_test_anomaly)),\r\n batch_size=batch_size)[:, t]\r\nanomaly_class_scores = anomaly_class_preds.mean(axis=-1)\r\n\r\n\r\ndef affine(x, is_flip, k_rotate):\r\n return tf.image.rot90(tf.image.flip_left_right(x) if is_flip else x,\r\n k=k_rotate)\r\n\r\n\r\nx_in = Input(batch_shape=mdl.input_shape)\r\ntransformations_sm_responses = [mdl(Lambda(affine, arguments={'is_flip': is_flip, 'k_rotate': k_rotate})(x_in))\r\n for is_flip, k_rotate in itertools.product((False, True), range(4))]\r\nout = average([Lambda(lambda sm_res: sm_res[:, j:j+1])(tens) for j, tens in enumerate(transformations_sm_responses)])\r\n\r\n\r\ninference_mdl = Model(x_in, out)\r\n\r\ngrads_tensor = K.gradients([inference_mdl.output], [inference_mdl.input])[0]\r\ngrads_fn = K.function([inference_mdl.input], [grads_tensor])\r\n\r\n\r\ndef optimize_anomaly_images():\r\n for im_ind in range(len(x_test_anomaly)):\r\n im = x_test_anomaly[im_ind:im_ind+1].copy()\r\n\r\n eta = 5\r\n for _ in range(200):\r\n grads = grads_fn([im])[0]\r\n grads[np.abs(grads * im) < np.percentile(np.abs(grads * im), 80)] = 0\r\n im_diff = grads * eta\r\n im_diff *= 0.99\r\n im += im_diff\r\n im = gaussian_filter(im, 0.28)\r\n\r\n im = np.clip(im, 0, 1)\r\n im[im < np.percentile(np.abs(im), 80)] = 0\r\n\r\n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(4, 2))\r\n ax1.imshow(x_test_anomaly[im_ind].squeeze(), cmap='Greys_r')\r\n ax1.grid(False)\r\n ax1.tick_params(which='both', bottom=False, top=False, left=False, right=False,\r\n labelbottom=False, labeltop=False, labelleft=False, labelright=False)\r\n ax2.imshow(im.squeeze(), cmap='Greys_r')\r\n ax2.grid(False)\r\n ax2.tick_params(which='both', bottom=False, top=False, left=False, right=False,\r\n labelbottom=False, labeltop=False, labelleft=False, labelright=False)\r\n\r\n fig.savefig('0_{}.png'.format(im_ind))\r\n plt.close()\r\n print('0_3_{} done'.format(im_ind))\r\n\r\n\r\ndef optimize_normal_images():\r\n for im_ind in range(len(x_train_single)):\r\n im = x_train_single[im_ind:im_ind+1].copy()\r\n\r\n eta = 5\r\n for _ in range(200):\r\n grads = grads_fn([im])[0]\r\n grads[np.abs(grads * im) < np.percentile(np.abs(grads * im), 80)] = 0\r\n im_diff = grads * eta\r\n im_diff *= 0.99\r\n im += im_diff\r\n im = gaussian_filter(im, 0.28)\r\n\r\n im = np.clip(im, 0, 1)\r\n im[im < np.percentile(np.abs(im), 80)] = 0\r\n\r\n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(4, 2))\r\n ax1.imshow(x_train_single[im_ind].squeeze(), cmap='Greys_r')\r\n ax1.grid(False)\r\n ax1.tick_params(which='both', bottom=False, top=False, left=False, right=False,\r\n labelbottom=False, labeltop=False, labelleft=False, labelright=False)\r\n ax2.imshow(im.squeeze(), cmap='Greys_r')\r\n ax2.grid(False)\r\n ax2.tick_params(which='both', bottom=False, top=False, left=False, right=False,\r\n labelbottom=False, labeltop=False, labelleft=False, labelright=False)\r\n\r\n fig.savefig('3_{}.png'.format(im_ind))\r\n plt.close()\r\n print('3_3_{} done'.format(im_ind))\r\n\r\n\r\noptimize_normal_images()\r\noptimize_anomaly_images()\r\n" }, { "path": "models/__init__.py", "content": "" }, { "path": "models/adgan.py", "content": "import numpy as np\r\nfrom keras.optimizers import Adam\r\nfrom keras.models import Model\r\nfrom keras.layers.merge import subtract\r\nfrom keras.utils.generic_utils import Progbar\r\nfrom keras.engine.topology import Input, Layer\r\nfrom keras.callbacks import CallbackList\r\nimport keras.backend as K\r\n\r\n\r\nclass GradPenLayer(Layer):\r\n def call(self, inputs, **kwargs):\r\n interp_in, critic_interp_score_in = inputs[0], inputs[1]\r\n interp_critic_grad = K.batch_flatten(K.gradients(critic_interp_score_in, [interp_in])[0])\r\n interp_critic_grad_norm = K.sqrt(K.sum(K.square(interp_critic_grad), axis=-1, keepdims=True))\r\n return K.square(interp_critic_grad_norm - 1.) # two sided regularisation\r\n # return K.square(K.relu(interp_critic_grad_norm - 1.)) # one sided regularisation\r\n\r\n def compute_output_shape(self, input_shape):\r\n return input_shape[0], 1\r\n\r\n\r\ndef train_wgan_with_grad_penalty(prior_gen, generator, data_gen, critic, batch_size, epochs,\r\n batches_per_epoch=100, optimizer=Adam(lr=1e-4, beta_1=0, beta_2=0.9),\r\n grad_pen_coef=10., critic_gen_train_ratio=2, callbacks=None):\r\n # build model to train the critic\r\n data_shape = critic.input_shape[1:]\r\n real_critic_input = Input(shape=data_shape, name='real_in')\r\n fake_critic_input = Input(shape=data_shape, name='fake_in')\r\n interp_critic_input = Input(shape=data_shape, name='interp_in')\r\n\r\n real_critic_score = critic(real_critic_input)\r\n fake_critic_score = critic(fake_critic_input)\r\n interp_critic_score = critic(interp_critic_input)\r\n\r\n critic_loss = subtract([fake_critic_score, real_critic_score])\r\n gradient_penalty = GradPenLayer()([interp_critic_input, interp_critic_score])\r\n\r\n critic_train_mdl = Model([real_critic_input, fake_critic_input, interp_critic_input],\r\n [critic_loss, gradient_penalty])\r\n\r\n critic_train_mdl.compile(optimizer=optimizer,\r\n loss=lambda y_true, y_pred: y_pred,\r\n loss_weights=[1., grad_pen_coef])\r\n\r\n # build model to train generator\r\n prior_input = Input(shape=generator.input_shape[1:], name='prior_in')\r\n critic.trainable = False\r\n critic_on_generator_score = critic(generator(prior_input))\r\n generator_train_mdl = Model(prior_input, critic_on_generator_score)\r\n generator_train_mdl.compile(optimizer=optimizer, loss=lambda y_true, y_pred: -y_pred)\r\n\r\n # init callbacks\r\n callbacks = callbacks or []\r\n callbacks = CallbackList(callbacks)\r\n callbacks.set_model({'generator': generator, 'critic': critic})\r\n callbacks.set_params({\r\n 'batch_size': batch_size,\r\n 'epochs': epochs,\r\n 'steps': batches_per_epoch,\r\n 'samples': batches_per_epoch * batch_size,\r\n 'prior_gen': prior_gen,\r\n 'data_gen': data_gen,\r\n })\r\n\r\n # train\r\n print('Training on {} samples for {} epochs'.format(batches_per_epoch * batch_size, epochs))\r\n callbacks.on_train_begin()\r\n for e in range(epochs):\r\n print('Epoch {}/{}'.format(e + 1, epochs))\r\n callbacks.on_epoch_begin(e)\r\n progbar = Progbar(target=batches_per_epoch*batch_size)\r\n dummy_y = np.array([None]*batch_size)\r\n for b in range(batches_per_epoch):\r\n callbacks.on_batch_begin(b)\r\n batch_losses = np.zeros(shape=3)\r\n for critic_upd in range(critic_gen_train_ratio):\r\n real_batch = data_gen(batch_size)\r\n fake_batch = generator.predict(prior_gen(batch_size))\r\n weights = np.random.uniform(size=batch_size)\r\n weights = weights.reshape((-1,) + (1,)*(len(real_batch.shape)-1))\r\n interp_batch = weights * real_batch + (1. - weights) * fake_batch\r\n\r\n x_batch = {'real_in': real_batch, 'fake_in': fake_batch, 'interp_in': interp_batch}\r\n cur_losses = np.array(critic_train_mdl.train_on_batch(x=x_batch, y=[dummy_y, dummy_y]))\r\n batch_losses += cur_losses\r\n\r\n generator_train_mdl.train_on_batch(x=prior_gen(batch_size), y=dummy_y)\r\n\r\n losses_names = ('total_loss', 'critic_loss', 'gradient_pen')\r\n progbar.add(batch_size, zip(losses_names, batch_losses))\r\n callbacks.on_batch_end(b)\r\n\r\n progbar.update(batches_per_epoch*batch_size)\r\n callbacks.on_epoch_end(e)\r\n\r\n callbacks.on_train_end()\r\n\r\n\r\ndef scores_from_adgan_generator(x_test, prior_gen, generator, n_seeds=8, k=5, z_lr=0.25, gen_lr=5e-5):\r\n generator.trainable = True\r\n initial_weights = generator.get_weights()\r\n\r\n gen_opt = Adam(lr=gen_lr, beta_1=0.5)\r\n z_opt = Adam(lr=z_lr, beta_1=0.5)\r\n\r\n x_ph = K.placeholder((1,)+x_test.shape[1:])\r\n z = K.variable(prior_gen(1))\r\n rec_loss = K.mean(K.square(x_ph - generator(z)))\r\n z_train_fn = K.function([x_ph], [rec_loss], updates=z_opt.get_updates(rec_loss, [z]))\r\n g_train_fn = K.function([x_ph, K.learning_phase()], [rec_loss],\r\n updates=gen_opt.get_updates(rec_loss, generator.trainable_weights))\r\n\r\n gen_opt_initial_params = gen_opt.get_weights()\r\n z_opt_initial_params = z_opt.get_weights()\r\n\r\n scores = []\r\n for x in x_test:\r\n x = np.expand_dims(x, axis=0)\r\n losses = []\r\n for j in range(n_seeds):\r\n K.set_value(z, prior_gen(1))\r\n generator.set_weights(initial_weights)\r\n gen_opt.set_weights(gen_opt_initial_params)\r\n z_opt.set_weights(z_opt_initial_params)\r\n for _ in range(k):\r\n z_train_fn([x])\r\n g_train_fn([x, 1])\r\n loss = z_train_fn([x])[0]\r\n losses.append(loss)\r\n\r\n score = -np.mean(losses)\r\n scores.append(score)\r\n\r\n return np.array(scores)\r\n" }, { "path": "models/dagmm.py", "content": "import numpy as np\r\nfrom keras.models import Input, Model\r\nfrom keras.layers import concatenate, Lambda, Layer, add\r\nimport keras.backend as K\r\nimport tensorflow as tf\r\n\r\n\r\nclass GaussianMixtureComponent(Layer):\r\n def __init__(self, lambd_diag=0.005, **kwargs):\r\n self.lambd_diag = lambd_diag\r\n super().__init__(**kwargs)\r\n\r\n def build(self, input_shapes):\r\n z_shape, _ = input_shapes\r\n self.phi = self.add_weight(name='phi',\r\n shape=(1,),\r\n initializer='ones',\r\n trainable=False)\r\n self.mu = self.add_weight(name='mu',\r\n shape=(1, z_shape[1]),\r\n initializer='uniform',\r\n trainable=False)\r\n self.sigma = self.add_weight(name='sig',\r\n shape=(z_shape[1], z_shape[1]),\r\n initializer='identity',\r\n trainable=False)\r\n super().build(input_shapes)\r\n\r\n def call(self, inputs, training=None):\r\n z, gamma_k = inputs\r\n\r\n gamma_k_sum = K.sum(gamma_k)\r\n est_phi = K.mean(gamma_k, axis=0)\r\n est_mu = K.dot(K.transpose(gamma_k), z) / gamma_k_sum\r\n est_sigma = K.dot(K.transpose(z - est_mu),\r\n gamma_k * (z - est_mu)) / gamma_k_sum\r\n\r\n est_sigma = est_sigma + (K.random_normal(shape=(K.int_shape(z)[1], 1), mean=1e-3, stddev=1e-4) * K.eye(K.int_shape(z)[1]))\r\n\r\n self.add_update(K.update(self.phi, est_phi), inputs)\r\n self.add_update(K.update(self.mu, est_mu), inputs)\r\n self.add_update(K.update(self.sigma, est_sigma), inputs)\r\n\r\n est_sigma_diag_inv = K.eye(K.int_shape(self.sigma)[0]) / est_sigma\r\n self.add_loss(self.lambd_diag * K.sum(est_sigma_diag_inv), inputs)\r\n\r\n phi = K.in_train_phase(est_phi, self.phi, training)\r\n mu = K.in_train_phase(est_mu, self.mu, training)\r\n sigma = K.in_train_phase(est_sigma, self.sigma, training)\r\n return GaussianMixtureComponent._calc_component_density(z, phi, mu, sigma)\r\n\r\n @staticmethod\r\n def _calc_component_density(z, phi, mu, sigma):\r\n sig_inv = tf.matrix_inverse(sigma)\r\n sig_sqrt_det = K.sqrt(tf.matrix_determinant(2 * np.pi * sigma) + K.epsilon())\r\n density = phi * (K.exp(-0.5 * K.sum(K.dot(z - mu, sig_inv) * (z - mu),\r\n axis=-1,\r\n keepdims=True)) / sig_sqrt_det) + K.epsilon()\r\n\r\n return density\r\n\r\n def compute_output_shape(self, input_shapes):\r\n z_shape, gamma_k_shape = input_shapes\r\n return z_shape[0], 1\r\n\r\n\r\ndef create_dagmm_model(encoder, decoder, estimation_encoder, lambd_diag=0.005):\r\n x_in = Input(batch_shape=encoder.input_shape)\r\n zc = encoder(x_in)\r\n\r\n decoder.name = 'reconstruction'\r\n x_rec = decoder(zc)\r\n euclid_dist = Lambda(lambda args: K.sqrt(K.sum(K.batch_flatten(K.square(args[0] - args[1])),\r\n axis=-1, keepdims=True) /\r\n K.sum(K.batch_flatten(K.square(args[0])),\r\n axis=-1, keepdims=True)),\r\n output_shape=(1,))([x_in, x_rec])\r\n cos_sim = Lambda(lambda args: K.batch_dot(K.l2_normalize(K.batch_flatten(args[0]), axis=-1),\r\n K.l2_normalize(K.batch_flatten(args[1]), axis=-1),\r\n axes=-1),\r\n output_shape=(1,))([x_in, x_rec])\r\n\r\n zr = concatenate([euclid_dist, cos_sim])\r\n z = concatenate([zc, zr])\r\n\r\n gamma = estimation_encoder(z)\r\n\r\n gamma_ks = [Lambda(lambda g: g[:, k:k + 1], output_shape=(1,))(gamma)\r\n for k in range(estimation_encoder.output_shape[-1])]\r\n\r\n components = [GaussianMixtureComponent(lambd_diag)([z, gamma_k])\r\n for gamma_k in gamma_ks]\r\n density = add(components) if len(components) > 1 else components[0]\r\n energy = Lambda(lambda dens: -K.log(dens), name='energy')(density)\r\n\r\n dagmm = Model(x_in, [x_rec, energy])\r\n\r\n return dagmm\r\n" }, { "path": "models/dsebm.py", "content": "from keras.layers import *\r\nfrom keras.models import Model\r\nimport keras.backend as K\r\n\r\n\r\nclass Prior(Layer):\r\n def __init__(self,\r\n bias_initializer='zeros',\r\n bias_regularizer=None,\r\n bias_constraint=None,\r\n **kwargs):\r\n if 'input_shape' not in kwargs and 'input_dim' in kwargs:\r\n kwargs['input_shape'] = (kwargs.pop('input_dim'),)\r\n super(Prior, self).__init__(**kwargs)\r\n self.bias_initializer = initializers.get(bias_initializer)\r\n self.bias_regularizer = regularizers.get(bias_regularizer)\r\n self.bias_constraint = constraints.get(bias_constraint)\r\n self.input_spec = InputSpec(min_ndim=2)\r\n self.bias = None\r\n\r\n def build(self, input_shape):\r\n self.bias = self.add_weight(shape=input_shape[1:],\r\n initializer=self.bias_initializer,\r\n name='bias',\r\n regularizer=self.bias_regularizer,\r\n constraint=self.bias_constraint)\r\n super(Prior, self).build(input_shape)\r\n\r\n def compute_output_shape(self, input_shape):\r\n return input_shape[0], 1\r\n\r\n def call(self, x, **kwargs):\r\n return K.sum(K.batch_flatten(K.square(K.bias_add(x, -self.bias))), axis=-1, keepdims=True)\r\n\r\n\r\ndef create_energy_model(encoder_mdl):\r\n x_in = Input(batch_shape=encoder_mdl.input_shape)\r\n\r\n encoded = encoder_mdl(x_in)\r\n prior = Prior()(x_in)\r\n energy = Lambda(lambda args: 0.5*args[0] - K.sum(K.batch_flatten(args[1]), axis=-1, keepdims=True),\r\n output_shape=(1,))([prior, encoded])\r\n return Model(x_in, energy)\r\n\r\n\r\ndef create_reconstruction_model(energy_mdl):\r\n x_in = Input(batch_shape=energy_mdl.input_shape)\r\n x = GaussianNoise(stddev=.5)(x_in) # only active in training\r\n energy = energy_mdl(x)\r\n rec = Lambda(lambda args: args[1] - K.gradients(args[0], args[1]),\r\n output_shape=energy_mdl.input_shape[1:])([energy, x])\r\n\r\n return Model(x_in, rec)\r\n" }, { "path": "models/encoders_decoders.py", "content": "from keras.models import Model\r\nfrom keras.layers import Input, Conv2D, Activation, BatchNormalization, Flatten, Dense, Conv2DTranspose, Reshape\r\nfrom utils import get_channels_axis\r\n\r\n\r\ndef conv_encoder(input_side=32, n_channels=3, representation_dim=256, representation_activation='tanh',\r\n intermediate_activation='relu'):\r\n nf = 64\r\n input_shape = (n_channels, input_side, input_side) if get_channels_axis() == 1 else (input_side, input_side,\r\n n_channels)\r\n\r\n x_in = Input(shape=input_shape)\r\n enc = x_in\r\n\r\n # downsample x0.5\r\n enc = Conv2D(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)\r\n enc = BatchNormalization(axis=get_channels_axis())(enc)\r\n enc = Activation(intermediate_activation)(enc)\r\n\r\n # downsample x0.5\r\n enc = Conv2D(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)\r\n enc = BatchNormalization(axis=get_channels_axis())(enc)\r\n enc = Activation(intermediate_activation)(enc)\r\n\r\n # downsample x0.5\r\n enc = Conv2D(nf * 4, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)\r\n enc = BatchNormalization(axis=get_channels_axis())(enc)\r\n enc = Activation(intermediate_activation)(enc)\r\n\r\n if input_side == 64:\r\n # downsample x0.5\r\n enc = Conv2D(nf * 8, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)\r\n enc = BatchNormalization(axis=get_channels_axis())(enc)\r\n enc = Activation(intermediate_activation)(enc)\r\n\r\n enc = Flatten()(enc)\r\n rep = Dense(representation_dim, activation=representation_activation)(enc)\r\n\r\n return Model(x_in, rep)\r\n\r\n\r\ndef conv_decoder(output_side=32, n_channels=3, representation_dim=256, activation='relu'):\r\n nf = 64\r\n\r\n rep_in = Input(shape=(representation_dim,))\r\n\r\n g = Dense(nf * 4 * 4 * 4)(rep_in)\r\n g = BatchNormalization(axis=-1)(g)\r\n g = Activation(activation)(g)\r\n\r\n conv_shape = (nf * 4, 4, 4) if get_channels_axis() == 1 else (4, 4, nf * 4)\r\n g = Reshape(conv_shape)(g)\r\n\r\n # upsample x2\r\n g = Conv2DTranspose(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)\r\n g = BatchNormalization(axis=get_channels_axis())(g)\r\n g = Activation(activation)(g)\r\n\r\n # upsample x2\r\n g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)\r\n g = BatchNormalization(axis=get_channels_axis())(g)\r\n g = Activation(activation)(g)\r\n\r\n if output_side == 64:\r\n # upsample x2\r\n g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)\r\n g = BatchNormalization(axis=get_channels_axis())(g)\r\n g = Activation(activation)(g)\r\n\r\n # upsample x2\r\n g = Conv2DTranspose(n_channels, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)\r\n g = Activation('tanh')(g)\r\n\r\n return Model(rep_in, g)\r\n\r\n\r\n" }, { "path": "models/wide_residual_network.py", "content": "from keras.models import Model\r\nfrom keras.layers import *\r\nfrom keras.regularizers import l2\r\nfrom keras.initializers import _compute_fans\r\nfrom keras.optimizers import SGD\r\nfrom keras import backend as K\r\n\r\n\r\nWEIGHT_DECAY = 0.5 * 0.0005\r\n\r\n\r\nclass SGDTorch(SGD):\r\n @interfaces.legacy_get_updates_support\r\n def get_updates(self, loss, params):\r\n grads = self.get_gradients(loss, params)\r\n self.updates = [K.update_add(self.iterations, 1)]\r\n\r\n lr = self.lr\r\n if self.initial_decay > 0:\r\n lr *= (1. / (1. + self.decay * K.cast(self.iterations,\r\n K.dtype(self.decay))))\r\n # momentum\r\n shapes = [K.int_shape(p) for p in params]\r\n moments = [K.zeros(shape) for shape in shapes]\r\n self.weights = [self.iterations] + moments\r\n for p, g, m in zip(params, grads, moments):\r\n v = self.momentum * m + g # velocity\r\n self.updates.append(K.update(m, v))\r\n\r\n if self.nesterov:\r\n new_p = p - lr * (self.momentum * v + g)\r\n else:\r\n new_p = p - lr * v\r\n\r\n # Apply constraints.\r\n if getattr(p, 'constraint', None) is not None:\r\n new_p = p.constraint(new_p)\r\n\r\n self.updates.append(K.update(p, new_p))\r\n return self.updates\r\n\r\n\r\ndef _get_channels_axis():\r\n return -1 if K.image_data_format() == 'channels_last' else 1\r\n\r\n\r\ndef _conv_kernel_initializer(shape, dtype=None):\r\n fan_in, fan_out = _compute_fans(shape)\r\n stddev = np.sqrt(2. / fan_in)\r\n return K.random_normal(shape, 0., stddev, dtype)\r\n\r\n\r\ndef _dense_kernel_initializer(shape, dtype=None):\r\n fan_in, fan_out = _compute_fans(shape)\r\n stddev = 1. / np.sqrt(fan_in)\r\n return K.random_uniform(shape, -stddev, stddev, dtype)\r\n\r\n\r\ndef batch_norm():\r\n return BatchNormalization(axis=_get_channels_axis(), momentum=0.9, epsilon=1e-5,\r\n beta_regularizer=l2(WEIGHT_DECAY), gamma_regularizer=l2(WEIGHT_DECAY))\r\n\r\n\r\ndef conv2d(output_channels, kernel_size, strides=1):\r\n return Convolution2D(output_channels, kernel_size, strides=strides, padding='same', use_bias=False,\r\n kernel_initializer=_conv_kernel_initializer, kernel_regularizer=l2(WEIGHT_DECAY))\r\n\r\n\r\ndef dense(output_units):\r\n return Dense(output_units, kernel_initializer=_dense_kernel_initializer, kernel_regularizer=l2(WEIGHT_DECAY),\r\n bias_regularizer=l2(WEIGHT_DECAY))\r\n\r\n\r\ndef _add_basic_block(x_in, out_channels, strides, dropout_rate=0.0):\r\n is_channels_equal = K.int_shape(x_in)[_get_channels_axis()] == out_channels\r\n\r\n bn1 = batch_norm()(x_in)\r\n bn1 = Activation('relu')(bn1)\r\n out = conv2d(out_channels, 3, strides)(bn1)\r\n out = batch_norm()(out)\r\n out = Activation('relu')(out)\r\n out = Dropout(dropout_rate)(out)\r\n out = conv2d(out_channels, 3, 1)(out)\r\n shortcut = x_in if is_channels_equal else conv2d(out_channels, 1, strides)(bn1)\r\n out = add([out, shortcut])\r\n return out\r\n\r\n\r\ndef _add_conv_group(x_in, out_channels, n, strides, dropout_rate=0.0):\r\n out = _add_basic_block(x_in, out_channels, strides, dropout_rate)\r\n for _ in range(1, n):\r\n out = _add_basic_block(out, out_channels, 1, dropout_rate)\r\n return out\r\n\r\n\r\ndef create_wide_residual_network(input_shape, num_classes, depth, widen_factor=1, dropout_rate=0.0,\r\n final_activation='softmax'):\r\n n_channels = [16, 16*widen_factor, 32*widen_factor, 64*widen_factor]\r\n assert ((depth - 4) % 6 == 0), 'depth should be 6n+4'\r\n n = (depth - 4) // 6\r\n\r\n inp = Input(shape=input_shape)\r\n conv1 = conv2d(n_channels[0], 3)(inp) # one conv at the beginning (spatial size: 32x32)\r\n conv2 = _add_conv_group(conv1, n_channels[1], n, 1, dropout_rate) # Stage 1 (spatial size: 32x32)\r\n conv3 = _add_conv_group(conv2, n_channels[2], n, 2, dropout_rate) # Stage 2 (spatial size: 16x16)\r\n conv4 = _add_conv_group(conv3, n_channels[3], n, 2, dropout_rate) # Stage 3 (spatial size: 8x8)\r\n\r\n out = batch_norm()(conv4)\r\n out = Activation('relu')(out)\r\n out = AveragePooling2D(8)(out)\r\n out = Flatten()(out)\r\n\r\n out = dense(num_classes)(out)\r\n out = Activation(final_activation)(out)\r\n\r\n return Model(inp, out)\r\n" }, { "path": "multiclass_experiment.py", "content": "from glob import glob\r\nimport os\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nfrom keras.utils import to_categorical\r\nfrom keras.preprocessing.image import ImageDataGenerator\r\nfrom keras.callbacks import LearningRateScheduler\r\nfrom keras.layers import Activation\r\nfrom keras import backend as K\r\nfrom keras.models import Model\r\nfrom models.wide_residual_network import create_wide_residual_network, SGDTorch, dense\r\nfrom utils import load_cifar10, normalize_minus1_1, save_roc_pr_curve_data\r\nfrom transformations import Transformer\r\n\r\n\r\ndef load_tinyimagenet(tinyimagenet_path='./'):\r\n images = [plt.imread(fp) for fp in glob(os.path.join(tinyimagenet_path, '*.jpg'))]\r\n\r\n for i in range(len(images)):\r\n if len(images[i].shape) != 3:\r\n images[i] = np.stack([images[i], images[i], images[i]], axis=-1)\r\n\r\n images = np.stack(images)\r\n images = normalize_minus1_1(K.cast_to_floatx(images))\r\n return images\r\n\r\n\r\ndef train_cifar10():\r\n (x_train, y_train), (x_test, y_test) = load_cifar10()\r\n\r\n idg = ImageDataGenerator(\r\n horizontal_flip=True,\r\n height_shift_range=4,\r\n width_shift_range=4,\r\n fill_mode='reflect'\r\n )\r\n\r\n idg.fit(x_train)\r\n\r\n n = 16\r\n k = 8\r\n mdl = create_wide_residual_network(x_train.shape[1:], 10, n, k)\r\n mdl.compile(SGDTorch(lr=.1, momentum=0.9, nesterov=True), 'categorical_crossentropy', ['acc'])\r\n\r\n lr_cb = LearningRateScheduler(lambda e: 0.1 * (0.2 ** (e >= 160 and 3 or e >= 120 and 2 or e >= 60 and 1 or 0)))\r\n\r\n batch_size = 128\r\n mdl.fit_generator(\r\n generator=idg.flow(x_train, to_categorical(y_train), batch_size=batch_size),\r\n epochs=200,\r\n validation_data=(idg.standardize(x_test), to_categorical(y_test)),\r\n callbacks=[lr_cb]\r\n )\r\n mdl.save_weights('cifar10_WRN_{}-{}.h5'.format(n, k))\r\n\r\n\r\ndef train_cifar10_transformations():\r\n (x_train, y_train), _ = load_cifar10()\r\n\r\n transformer = Transformer(8, 8)\r\n\r\n def data_gen(x, y, batch_size):\r\n while True:\r\n ind_permutation = np.random.permutation(len(x))\r\n for b_start_ind in range(0, len(x), batch_size):\r\n batch_inds = ind_permutation[b_start_ind:b_start_ind + batch_size]\r\n x_batch = x[batch_inds]\r\n y_batch = y[batch_inds].flatten()\r\n\r\n if K.image_data_format() == 'channels_first':\r\n x_batch = np.transpose(x_batch, (0, 2, 3, 1))\r\n\r\n y_t_batch = np.random.randint(0, transformer.n_transforms, size=len(x_batch))\r\n\r\n x_batch = transformer.transform_batch(x_batch, y_t_batch)\r\n\r\n if K.image_data_format() == 'channels_first':\r\n x_batch = np.transpose(x_batch, (0, 3, 1, 2))\r\n\r\n yield (x_batch, [to_categorical(y_batch, num_classes=10), to_categorical(y_t_batch, num_classes=transformer.n_transforms)])\r\n\r\n n = 16\r\n k = 8\r\n base_mdl = create_wide_residual_network(x_train.shape[1:], 10, n, k)\r\n\r\n transformations_cls_out = Activation('softmax')(dense(transformer.n_transforms)(base_mdl.get_layer(index=-3).output))\r\n\r\n mdl = Model(base_mdl.input, [base_mdl.output, transformations_cls_out])\r\n\r\n mdl.compile(SGDTorch(lr=.1, momentum=0.9, nesterov=True), 'categorical_crossentropy', ['acc'])\r\n\r\n lr_cb = LearningRateScheduler(lambda e: 0.1 * (0.2 ** (e >= 160 and 3 or e >= 120 and 2 or e >= 60 and 1 or 0)))\r\n\r\n batch_size = 128\r\n mdl.fit_generator(\r\n generator=data_gen(x_train, y_train, batch_size=batch_size),\r\n steps_per_epoch=len(x_train) // batch_size,\r\n epochs=200,\r\n callbacks=[lr_cb]\r\n )\r\n mdl.save_weights('cifar10_WRN_doublehead-transformations_{}-{}.h5'.format(n, k))\r\n\r\n\r\ndef transformation_cifar10_vs_tinyimagenet():\r\n _, (x_test, y_test) = load_cifar10()\r\n x_test_out = load_tinyimagenet('/home/izikgo/Imagenet_resize/Imagenet_resize/')\r\n\r\n transformer = Transformer(8, 8)\r\n n = 16\r\n k = 8\r\n base_mdl = create_wide_residual_network(x_test.shape[1:], 10, n, k)\r\n\r\n transformations_cls_out = Activation('softmax')(dense(transformer.n_transforms)(base_mdl.get_layer(index=-3).output))\r\n\r\n mdl = Model(base_mdl.input, [base_mdl.output, transformations_cls_out])\r\n mdl.load_weights('cifar10_WRN_doublehead-transformations_{}-{}.h5'.format(n, k))\r\n\r\n scores_mdl = Model(mdl.input, mdl.output[1])\r\n x_test_all = np.concatenate((x_test, x_test_out))\r\n preds = np.zeros((len(x_test_all), transformer.n_transforms))\r\n for t in range(transformer.n_transforms):\r\n preds[:, t] = scores_mdl.predict(transformer.transform_batch(x_test_all, [t] * len(x_test_all)),\r\n batch_size=128)[:, t]\r\n\r\n labels = np.concatenate((np.ones(len(x_test)), np.zeros(len(x_test_out))))\r\n scores = preds.mean(axis=-1)\r\n\r\n save_roc_pr_curve_data(scores, labels, 'cifar10-vs-tinyimagenet_transformations.npz')\r\n\r\n\r\nif __name__ == '__main__':\r\n train_cifar10_transformations()\r\n transformation_cifar10_vs_tinyimagenet()\r\n" }, { "path": "transformations.py", "content": "import abc\r\nimport itertools\r\nimport numpy as np\r\n\r\nfrom keras.preprocessing.image import apply_affine_transform\r\n\r\n\r\nclass AffineTransformation(object):\r\n def __init__(self, flip, tx, ty, k_90_rotate):\r\n self.flip = flip\r\n self.tx = tx\r\n self.ty = ty\r\n self.k_90_rotate = k_90_rotate\r\n\r\n def __call__(self, x):\r\n res_x = x\r\n if self.flip:\r\n res_x = np.fliplr(res_x)\r\n if self.tx != 0 or self.ty != 0:\r\n res_x = apply_affine_transform(res_x, tx=self.tx, ty=self.ty, channel_axis=2, fill_mode='reflect')\r\n if self.k_90_rotate != 0:\r\n res_x = np.rot90(res_x, self.k_90_rotate)\r\n\r\n return res_x\r\n\r\n\r\nclass AbstractTransformer(abc.ABC):\r\n def __init__(self):\r\n self._transformation_list = None\r\n self._create_transformation_list()\r\n\r\n @property\r\n def n_transforms(self):\r\n return len(self._transformation_list)\r\n\r\n @abc.abstractmethod\r\n def _create_transformation_list(self):\r\n return\r\n\r\n def transform_batch(self, x_batch, t_inds):\r\n assert len(x_batch) == len(t_inds)\r\n\r\n transformed_batch = x_batch.copy()\r\n for i, t_ind in enumerate(t_inds):\r\n transformed_batch[i] = self._transformation_list[t_ind](transformed_batch[i])\r\n return transformed_batch\r\n\r\n\r\nclass Transformer(AbstractTransformer):\r\n def __init__(self, translation_x=8, translation_y=8):\r\n self.max_tx = translation_x\r\n self.max_ty = translation_y\r\n super().__init__()\r\n\r\n def _create_transformation_list(self):\r\n transformation_list = []\r\n for is_flip, tx, ty, k_rotate in itertools.product((False, True),\r\n (0, -self.max_tx, self.max_tx),\r\n (0, -self.max_ty, self.max_ty),\r\n range(4)):\r\n transformation = AffineTransformation(is_flip, tx, ty, k_rotate)\r\n transformation_list.append(transformation)\r\n\r\n self._transformation_list = transformation_list\r\n\r\n\r\nclass SimpleTransformer(AbstractTransformer):\r\n def _create_transformation_list(self):\r\n transformation_list = []\r\n for is_flip, k_rotate in itertools.product((False, True),\r\n range(4)):\r\n transformation = AffineTransformation(is_flip, 0, 0, k_rotate)\r\n transformation_list.append(transformation)\r\n\r\n self._transformation_list = transformation_list\r\n\r\n" }, { "path": "utils.py", "content": "from glob import glob\r\nimport os\r\nimport numpy as np\r\nimport cv2\r\nfrom sklearn.metrics import roc_curve, precision_recall_curve, auc\r\nfrom keras.datasets import mnist, fashion_mnist, cifar100, cifar10\r\nfrom keras.backend import cast_to_floatx\r\n\r\n\r\ndef resize_and_crop_image(input_file, output_side_length, greyscale=False):\r\n img = cv2.imread(input_file)\r\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB if not greyscale else cv2.COLOR_BGR2GRAY)\r\n height, width = img.shape[:2]\r\n new_height = output_side_length\r\n new_width = output_side_length\r\n if height > width:\r\n new_height = int(output_side_length * height / width)\r\n else:\r\n new_width = int(output_side_length * width / height)\r\n resized_img = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_AREA)\r\n height_offset = (new_height - output_side_length) // 2\r\n width_offset = (new_width - output_side_length) // 2\r\n cropped_img = resized_img[height_offset:height_offset + output_side_length,\r\n width_offset:width_offset + output_side_length]\r\n assert cropped_img.shape[:2] == (output_side_length, output_side_length)\r\n return cropped_img\r\n\r\n\r\ndef normalize_minus1_1(data):\r\n return 2*(data/255.) - 1\r\n\r\n\r\ndef get_channels_axis():\r\n import keras\r\n idf = keras.backend.image_data_format()\r\n if idf == 'channels_first':\r\n return 1\r\n assert idf == 'channels_last'\r\n return 3\r\n\r\n\r\ndef load_fashion_mnist():\r\n (X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()\r\n X_train = normalize_minus1_1(cast_to_floatx(np.pad(X_train, ((0, 0), (2, 2), (2, 2)), 'constant')))\r\n X_train = np.expand_dims(X_train, axis=get_channels_axis())\r\n X_test = normalize_minus1_1(cast_to_floatx(np.pad(X_test, ((0, 0), (2, 2), (2, 2)), 'constant')))\r\n X_test = np.expand_dims(X_test, axis=get_channels_axis())\r\n return (X_train, y_train), (X_test, y_test)\r\n\r\n\r\ndef load_mnist():\r\n (X_train, y_train), (X_test, y_test) = mnist.load_data()\r\n X_train = normalize_minus1_1(cast_to_floatx(np.pad(X_train, ((0, 0), (2, 2), (2, 2)), 'constant')))\r\n X_train = np.expand_dims(X_train, axis=get_channels_axis())\r\n X_test = normalize_minus1_1(cast_to_floatx(np.pad(X_test, ((0, 0), (2, 2), (2, 2)), 'constant')))\r\n X_test = np.expand_dims(X_test, axis=get_channels_axis())\r\n return (X_train, y_train), (X_test, y_test)\r\n\r\n\r\ndef load_cifar10():\r\n (X_train, y_train), (X_test, y_test) = cifar10.load_data()\r\n X_train = normalize_minus1_1(cast_to_floatx(X_train))\r\n X_test = normalize_minus1_1(cast_to_floatx(X_test))\r\n return (X_train, y_train), (X_test, y_test)\r\n\r\n\r\ndef load_cifar100(label_mode='coarse'):\r\n (X_train, y_train), (X_test, y_test) = cifar100.load_data(label_mode=label_mode)\r\n X_train = normalize_minus1_1(cast_to_floatx(X_train))\r\n X_test = normalize_minus1_1(cast_to_floatx(X_test))\r\n return (X_train, y_train), (X_test, y_test)\r\n\r\n\r\ndef save_roc_pr_curve_data(scores, labels, file_path):\r\n scores = scores.flatten()\r\n labels = labels.flatten()\r\n\r\n scores_pos = scores[labels == 1]\r\n scores_neg = scores[labels != 1]\r\n\r\n truth = np.concatenate((np.zeros_like(scores_neg), np.ones_like(scores_pos)))\r\n preds = np.concatenate((scores_neg, scores_pos))\r\n fpr, tpr, roc_thresholds = roc_curve(truth, preds)\r\n roc_auc = auc(fpr, tpr)\r\n\r\n # pr curve where \"normal\" is the positive class\r\n precision_norm, recall_norm, pr_thresholds_norm = precision_recall_curve(truth, preds)\r\n pr_auc_norm = auc(recall_norm, precision_norm)\r\n\r\n # pr curve where \"anomaly\" is the positive class\r\n precision_anom, recall_anom, pr_thresholds_anom = precision_recall_curve(truth, -preds, pos_label=0)\r\n pr_auc_anom = auc(recall_anom, precision_anom)\r\n\r\n np.savez_compressed(file_path,\r\n preds=preds, truth=truth,\r\n fpr=fpr, tpr=tpr, roc_thresholds=roc_thresholds, roc_auc=roc_auc,\r\n precision_norm=precision_norm, recall_norm=recall_norm,\r\n pr_thresholds_norm=pr_thresholds_norm, pr_auc_norm=pr_auc_norm,\r\n precision_anom=precision_anom, recall_anom=recall_anom,\r\n pr_thresholds_anom=pr_thresholds_anom, pr_auc_anom=pr_auc_anom)\r\n\r\n\r\ndef create_cats_vs_dogs_npz(cats_vs_dogs_path='./'):\r\n labels = ['cat', 'dog']\r\n label_to_y_dict = {l: i for i, l in enumerate(labels)}\r\n\r\n def _load_from_dir(dir_name):\r\n glob_path = os.path.join(cats_vs_dogs_path, dir_name, '*.*.jpg')\r\n imgs_paths = glob(glob_path)\r\n images = [resize_and_crop_image(p, 64) for p in imgs_paths]\r\n x = np.stack(images)\r\n y = [label_to_y_dict[os.path.split(p)[-1][:3]] for p in imgs_paths]\r\n y = np.array(y)\r\n return x, y\r\n\r\n x_train, y_train = _load_from_dir('train')\r\n x_test, y_test = _load_from_dir('test')\r\n\r\n np.savez_compressed(os.path.join(cats_vs_dogs_path, 'cats_vs_dogs.npz'),\r\n x_train=x_train, y_train=y_train,\r\n x_test=x_test, y_test=y_test)\r\n\r\n\r\ndef load_cats_vs_dogs(cats_vs_dogs_path='./'):\r\n npz_file = np.load(os.path.join(cats_vs_dogs_path, 'cats_vs_dogs.npz'))\r\n x_train = normalize_minus1_1(cast_to_floatx(npz_file['x_train']))\r\n y_train = npz_file['y_train']\r\n x_test = normalize_minus1_1(cast_to_floatx(npz_file['x_test']))\r\n y_test = npz_file['y_test']\r\n\r\n return (x_train, y_train), (x_test, y_test)\r\n\r\n\r\ndef get_class_name_from_index(index, dataset_name):\r\n ind_to_name = {\r\n 'cifar10': ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'),\r\n 'cifar100': ('aquatic mammals', 'fish', 'flowers', 'food containers', 'fruit and vegetables',\r\n 'household electrical devices', 'household furniture', 'insects', 'large carnivores',\r\n 'large man-made outdoor things', 'large natural outdoor scenes', 'large omnivores and herbivores',\r\n 'medium-sized mammals', 'non-insect invertebrates', 'people', 'reptiles', 'small mammals', 'trees',\r\n 'vehicles 1', 'vehicles 2'),\r\n 'fashion-mnist': ('t-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag',\r\n 'ankle-boot'),\r\n 'cats-vs-dogs': ('cat', 'dog'),\r\n }\r\n\r\n return ind_to_name[dataset_name][index]\r\n" } ]