[ { "path": "GrayScott2D/Gray_Scott.py", "content": "import numpy as np\r\nimport matplotlib.pyplot as plt\r\nimport scipy.io as sio\r\nfrom scipy.interpolate import griddata\r\nfrom models_tf import Sampler, ResidualSampler, DataSampler, Gray_Scott2D\r\n\r\nif __name__ == '__main__':\r\n # Reload data\r\n datafile = 'data.npy'\r\n data = np.load(datafile, allow_pickle=True).item()\r\n\r\n X = data['X']\r\n U = data['U']\r\n\r\n # Time intervals\r\n tspan = data['tspan']\r\n T1 = data['T1']\r\n T2 = data['T2']\r\n\r\n # Parameters\r\n epsilon1 = data['ep1']\r\n epsilon2 = data['ep2']\r\n b = data['b']\r\n d = data['d']\r\n\r\n # Define data sampler and residual sampler\r\n dom_coords = np.array([[T1, -1.0, -1.0],\r\n [T2, 1.0, 1.0]])\r\n res_sampler = Sampler(3, dom_coords, lambda x: np.zeros_like(x), name='Forcing')\r\n\r\n data_sampler = DataSampler(X, U)\r\n\r\n # Create model\r\n layers = [3, 100, 100, 100, 100, 100, 100, 100, 2]\r\n model = Gray_Scott2D(data_sampler, res_sampler, layers, b, d)\r\n\r\n # Train model\r\n model.train(nIter=120000, batch_size=1000)\r\n\r\n # Save results\r\n model.saver.save(model.sess, 'SavedModels/' 'GS_param' + '_7x100_it120000' + '.ckpt')\r\n\r\n ep1 = model.sess.run(model.epsilon1)\r\n ep2 = model.sess.run(model.epsilon2)\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1), np.exp(ep2)))\r\n\r\n ep1_log = model.ep1_log\r\n ep2_log = model.ep2_log\r\n\r\n np.savetxt('SavedResults/' + 'ep1_log_original', ep1_log, delimiter=',')\r\n np.savetxt('SavedResults/' + 'ep2_log_original', ep2_log, delimiter=',')\r\n\r\n # Prediction\r\n raw_data = sio.loadmat('sol.mat')\r\n\r\n X = raw_data['X']\r\n Y = raw_data['Y']\r\n tspan = raw_data['tspan'].flatten()\r\n usol = raw_data['usol']\r\n vsol = raw_data['vsol']\r\n\r\n step = -50\r\n x = X.flatten()[:, None]\r\n y = Y.flatten()[:, None]\r\n t = tspan[step] * np.ones_like(x)\r\n\r\n X_star = np.concatenate([t, x, y], axis=1)\r\n\r\n u_pred, v_pred = model.predict(X_star)\r\n u_star = usol[:, :, step].flatten()[:, None]\r\n v_star = vsol[:, :, step].flatten()[:, None]\r\n\r\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\r\n error_v = np.linalg.norm(v_star - v_pred, 2) / np.linalg.norm(v_star, 2)\r\n\r\n print('Relative L2 error_u: %e' % (error_u))\r\n print('Relative L2 error_v: %e' % (error_v))\r\n\r\n ep1 = model.sess.run(model.epsilon1)\r\n ep2 = model.sess.run(model.epsilon2)\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1), np.exp(ep2)))\r\n\r\n # Plot\r\n U_star = griddata(np.concatenate([x, y], axis=1), u_pred.flatten(), (X, Y), method='cubic')\r\n V_star = griddata(np.concatenate([x, y], axis=1), v_pred.flatten(), (X, Y), method='cubic')\r\n\r\n plt.figure(figsize=(18, 5))\r\n plt.subplot(1, 3, 1)\r\n plt.pcolor(X, Y, U_star)\r\n plt.colorbar()\r\n plt.subplot(1, 3, 2)\r\n plt.pcolor(X, Y, usol[:, :, step])\r\n plt.colorbar()\r\n\r\n plt.subplot(1, 3, 3)\r\n plt.pcolor(X, Y, U_star - usol[:, :, step])\r\n plt.colorbar()\r\n plt.show()\r\n\r\n plt.figure(figsize=(18, 5))\r\n plt.subplot(1, 3, 1)\r\n plt.pcolor(X, Y, V_star)\r\n plt.colorbar()\r\n plt.subplot(1, 3, 2)\r\n plt.pcolor(X, Y, vsol[:, :, step])\r\n plt.colorbar()\r\n\r\n plt.subplot(1, 3, 3)\r\n plt.pcolor(X, Y, V_star - vsol[:, :, step])\r\n plt.colorbar()\r\n plt.show()\r\n\r\n\r\n" }, { "path": "GrayScott2D/Gray_Scott_FF.py", "content": "import numpy as np\nimport matplotlib.pyplot as plt\nimport scipy.io as sio\nfrom scipy.interpolate import griddata\nfrom models_tf import Sampler, ResidualSampler, DataSampler, Gray_Scott2D_FF\n\nif __name__ == '__main__':\n # Reload data\n datafile = 'data.npy'\n data = np.load(datafile, allow_pickle=True).item()\n\n X = data['X']\n U = data['U']\n\n # Time intervals\n tspan = data['tspan']\n T1 = data['T1']\n T2 = data['T2']\n\n # Parameters\n epsilon1 = data['ep1']\n epsilon2 = data['ep2']\n b = data['b']\n d = data['d']\n\n # Define data sampler and residual sampler\n dom_coords = np.array([[T1, -1.0, -1.0],\n [T2, 1.0, 1.0]])\n\n res_sampler = Sampler(3, dom_coords, lambda x: np.zeros_like(x), name='Forcing')\n\n data_sampler = DataSampler(X, U)\n\n # Create model\n layers = [100, 100, 100, 100, 100, 100, 100, 2]\n model = Gray_Scott2D_FF(data_sampler, res_sampler, layers, b, d)\n\n model.train(nIter=120000, batch_size=1000)\n\n model.saver.save(model.sess, 'SavedModels/' 'GS_param_FF' + '_7x100_it120000' + '.ckpt')\n\n ep1 = model.sess.run(model.epsilon1)\n ep2 = model.sess.run(model.epsilon2)\n\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1), np.exp(ep2)))\n\n ep1_log = model.ep1_log\n ep2_log = model.ep2_log\n\n loss_data = model.loss_u_log\n loss_res = model.loss_r_log\n\n np.savetxt('SavedResults/' + 'ep1_log', ep1_log, delimiter=',')\n np.savetxt('SavedResults/' + 'ep2_log', ep2_log, delimiter=',')\n\n np.savetxt('SavedResults/' + 'loss_data', loss_data, delimiter=',')\n np.savetxt('SavedResults/' + 'loss_res', loss_res, delimiter=',')\n\n # Prediction\n raw_data = sio.loadmat('sol.mat')\n\n X = raw_data['X']\n Y = raw_data['Y']\n tspan = raw_data['tspan'].flatten()\n usol = raw_data['usol']\n vsol = raw_data['vsol']\n\n step = -50\n x = X.flatten()[:, None]\n y = Y.flatten()[:, None]\n t = tspan[step] * np.ones_like(x)\n\n X_star = np.concatenate([t, x, y], axis=1)\n\n u_pred, v_pred = model.predict(X_star)\n u_star = usol[:, :, step].flatten()[:, None]\n v_star = vsol[:, :, step].flatten()[:, None]\n\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\n error_v = np.linalg.norm(v_star - v_pred, 2) / np.linalg.norm(v_star, 2)\n\n print('Relative L2 error_u: %e' % (error_u))\n print('Relative L2 error_v: %e' % (error_v))\n\n ep1 = model.sess.run(model.epsilon1)\n ep2 = model.sess.run(model.epsilon2)\n\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1), np.exp(ep2)))\n\n # Plot\n U_star = griddata(np.concatenate([x, y], axis=1), u_pred.flatten(), (X, Y), method='cubic')\n V_star = griddata(np.concatenate([x, y], axis=1), v_pred.flatten(), (X, Y), method='cubic')\n\n plt.figure(figsize=(18, 5))\n plt.subplot(1, 3, 1)\n plt.pcolor(X, Y, usol[:, :, step])\n # quadmesh = plt.pcolormesh(X,Y,usol[:,:,step])\n # quadmesh.set_clim(vmin=0, vmax=1)\n plt.colorbar()\n plt.title('Reference')\n\n plt.subplot(1, 3, 2)\n plt.pcolor(X, Y, U_star)\n plt.colorbar()\n plt.title('Predicted')\n\n plt.subplot(1, 3, 3)\n plt.pcolor(X, Y, U_star - usol[:, :, step])\n plt.colorbar()\n plt.title('Point-wise error')\n plt.show()\n\n plt.figure(figsize=(18, 5))\n\n plt.subplot(1, 3, 1)\n plt.pcolor(X, Y, vsol[:, :, step])\n plt.colorbar()\n plt.title('Reference')\n\n plt.subplot(1, 3, 2)\n plt.pcolor(X, Y, V_star)\n plt.colorbar()\n plt.title('Predicted')\n\n plt.subplot(1, 3, 3)\n plt.pcolor(X, Y, V_star - vsol[:, :, step])\n plt.colorbar()\n plt.title('Point-wise error')\n plt.show()\n\n plt.plot(ep1_log, label='ep1')\n plt.plot(ep2_log, label='ep2')\n plt.legend()\n plt.yscale('log')\n\n" }, { "path": "GrayScott2D/Gray_Scott_mFF.py", "content": "import numpy as np\r\nimport matplotlib.pyplot as plt\r\nimport scipy.io as sio\r\nfrom scipy.interpolate import griddata\r\nfrom models_tf import Sampler, ResidualSampler, DataSampler, Gray_Scott2D_ST_mFF, Gray_Scott2D_ST_mFF_adaptive\r\n\r\nif __name__ == '__main__':\r\n # Reload data\r\n datafile = 'data.npy'\r\n data = np.load(datafile, allow_pickle=True).item()\r\n\r\n X = data['X']\r\n U = data['U']\r\n\r\n # Time intervals\r\n tspan = data['tspan']\r\n T1 = data['T1']\r\n T2 = data['T2']\r\n\r\n # Parameters\r\n epsilon1 = data['ep1']\r\n epsilon2 = data['ep2']\r\n b = data['b']\r\n d = data['d']\r\n\r\n # Define data sampler and residual sampler\r\n dom_coords = np.array([[T1, -1.0, -1.0],\r\n [T2, 1.0, 1.0]])\r\n res_sampler = Sampler(3, dom_coords, lambda x: np.zeros_like(x), name='Forcing')\r\n\r\n data_sampler = DataSampler(X, U)\r\n\r\n # Create model\r\n layers = [100, 100, 100, 100, 100, 100, 100, 2]\r\n model = Gray_Scott2D_ST_mFF(data_sampler, res_sampler, layers, b, d)\r\n\r\n # Train model\r\n model.train(nIter=120000, batch_size=1000)\r\n\r\n # Save results\r\n model.saver.save(model.sess, 'SavedModels/' 'GS_param_ST_mFF' + '_7x100_it120000' + '.ckpt')\r\n\r\n ep1 = model.sess.run(model.epsilon1)\r\n ep2 = model.sess.run(model.epsilon2)\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1), np.exp(ep2)))\r\n\r\n ep1_log = model.ep1_log\r\n ep2_log = model.ep2_log\r\n\r\n loss_data = model.loss_u_log\r\n loss_res = model.loss_r_log\r\n\r\n np.savetxt('SavedResults/' + 'ep1_log', ep1_log, delimiter=',')\r\n np.savetxt('SavedResults/' + 'ep2_log', ep2_log, delimiter=',')\r\n\r\n np.savetxt('SavedResults/' + 'loss_data', loss_data, delimiter=',')\r\n np.savetxt('SavedResults/' + 'loss_res', loss_res, delimiter=',')\r\n\r\n # Prediction\r\n raw_data = sio.loadmat('sol.mat')\r\n\r\n X = raw_data['X']\r\n Y = raw_data['Y']\r\n tspan = raw_data['tspan'].flatten()\r\n usol = raw_data['usol']\r\n vsol = raw_data['vsol']\r\n\r\n step = -50\r\n x = X.flatten()[:, None]\r\n y = Y.flatten()[:, None]\r\n t = tspan[step] * np.ones_like(x)\r\n\r\n X_star = np.concatenate([t, x, y], axis=1)\r\n\r\n u_pred, v_pred = model.predict(X_star)\r\n u_star = usol[:, :, step].flatten()[:, None]\r\n v_star = vsol[:, :, step].flatten()[:, None]\r\n\r\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\r\n error_v = np.linalg.norm(v_star - v_pred, 2) / np.linalg.norm(v_star, 2)\r\n\r\n print('Relative L2 error_u: %e' % (error_u))\r\n print('Relative L2 error_v: %e' % (error_v))\r\n\r\n # Plot\r\n U_star = griddata(np.concatenate([x, y], axis=1), u_pred.flatten(), (X, Y), method='cubic')\r\n V_star = griddata(np.concatenate([x, y], axis=1), v_pred.flatten(), (X, Y), method='cubic')\r\n\r\n plt.figure(figsize=(18, 5))\r\n plt.subplot(1, 3, 1)\r\n plt.pcolor(X, Y, U_star)\r\n plt.colorbar()\r\n plt.subplot(1, 3, 2)\r\n plt.pcolor(X, Y, usol[:, :, step])\r\n plt.colorbar()\r\n\r\n plt.subplot(1, 3, 3)\r\n plt.pcolor(X, Y, U_star - usol[:, :, step])\r\n plt.colorbar()\r\n plt.show()\r\n\r\n plt.figure(figsize=(18, 5))\r\n plt.subplot(1, 3, 1)\r\n plt.pcolor(X, Y, V_star)\r\n plt.colorbar()\r\n plt.subplot(1, 3, 2)\r\n plt.pcolor(X, Y, vsol[:, :, step])\r\n plt.colorbar()\r\n\r\n plt.subplot(1, 3, 3)\r\n plt.pcolor(X, Y, V_star - vsol[:, :, step])\r\n plt.colorbar()\r\n plt.show()\r\n\r\n\r\n\r\n" }, { "path": "GrayScott2D/data/GrayScott.m", "content": "%% Gray-Scott equations in 2D\r\n% Nick Trefethen, April 2016\r\n\r\n%%\r\n% (Chebfun Example pde/GrayScott.m)\r\n% [Tags: #Gray-Scott, #spin2]\r\n\r\n%% 1. Rolls\r\n% The Gray-Scott equations are a pair of coupled reaction-diffusion\r\n% equations that lead to interesting patterns [1,2,3].\r\n% Let us look at two examples in 2D.\r\n\r\n%%\r\n% The equations are\r\n% $$ u_t = \\varepsilon_1\\Delta u + b(1-u) - uv^2, \\quad\r\n% v_t = \\varepsilon_2\\Delta v - dv + uv^2, $$\r\n% where $\\Delta$ is the Laplacian and $\\varepsilon_1,\r\n% \\varepsilon_2,b,d$ are parameters.\r\n% To begin with we choose these values.\r\nep1 = 0.00001; ep2 = 0.000005;\r\nb = 0.04; d = 0.1;\r\n%%\r\n% We now solve up to $t=3500$ with `spin2` and plot the $v$ variable.\r\n% What beautiful, random-seeming \"rolls\" (or\r\n% \"fingerprints\") appear! \r\nnn = 400;\r\nsteps = 500;\r\ndt = 0.5;\r\n\r\ndom = [-1 1 -1 1]; x = chebfun('x',dom(1:2)); tspan = linspace(0,5000, steps+1);\r\nS = spinop2(dom,tspan);\r\nS.lin = @(u,v) [ep1*lap(u); ep2*lap(v)];\r\nS.nonlin = @(u,v) [b*(1-u)-u.*v.^2;-d*v+u.*v.^2];\r\nS.init = chebfun2v(@(x,y) 1-exp(-80*((x+.05).^2+(y+.02).^2)), ...\r\n @(x,y) exp(-80*((x-.05).^2+(y-.02).^2)),dom);\r\ntic, u = spin2(S, nn, dt,'plot','off');\r\n\r\n% plot(u{1, 4}), view(0,90), axis equal, axis off\r\ntime_in_seconds = toc\r\n\r\nN = 200;\r\n[X,Y] = meshgrid(linspace(-1,1, N), linspace(-1,1, N));\r\n\r\nusol = zeros(N, N, steps+1);\r\nfor i = 1:steps+1\r\n usol(:,:,i) = u{1, i}(X,Y);\r\nend\r\n\r\nvsol = zeros(N,N, steps+1);\r\nfor i = 1:steps+1\r\n vsol(:,:,i) = u{2, i}(X,Y);\r\nend\r\n\r\n% save('sol.mat', 'b', 'd', 'ep1', 'ep2', 'tspan', 'usol', 'vsol', 'X', 'Y')\r\n" }, { "path": "GrayScott2D/data/parse_data.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Fri Sep 25 13:10:47 2020\r\n\r\n@author: Wsf12\r\n\"\"\"\r\n\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nimport scipy.io as sio\r\n\r\ndata = sio.loadmat('sol.mat')\r\n\r\nX = data['X']\r\nY = data['Y']\r\n\r\ntspan = data['tspan'].flatten()\r\n\r\nusol = data['usol']\r\nvsol = data['vsol']\r\n\r\nepsilon1 = data['ep1']\r\nepsilon2 = data['ep2']\r\nb = data['b']\r\nd = data['d']\r\n\r\nX_list = []\r\nU_list = []\r\n\r\nsteps = len(tspan) # 500 snap shots\r\n\r\nx = X.flatten()[:,None]\r\ny = Y.flatten()[:,None]\r\n\r\nT1 = 350\r\nT2 = 400 \r\n\r\nfor k in range(T1, T2 + 1):\r\n t = tspan[k] * np.ones_like(x) \r\n \r\n u = usol[:,:,k].flatten()[:, None]\r\n v = vsol[:,:,k].flatten()[:, None]\r\n \r\n X_list.append(np.concatenate([t, x, y], axis = 1))\r\n U_list.append(np.concatenate([u, v], axis = 1))\r\n\r\nX = np.vstack(X_list)\r\nU = np.vstack(U_list)\r\n\r\ndata_dict = {'X': X, 'U': U, \r\n 'tspan': tspan, 'T1': tspan[T1], 'T2': tspan[T2], \r\n 'ep1':epsilon1, 'ep2':epsilon2, 'b':b, 'd':d}\r\n\r\nnp.save('data.npy', data_dict)\r\n\r\n\r\n## data down sampling\r\n#X_reduced = data['X_reduced']\r\n#Y_reduced = data['Y_reduced']\r\n#\r\n#tspan = data['tspan'].flatten()\r\n#\r\n#usol = data['usol_reduced']\r\n#vsol = data['vsol_reduced']\r\n#\r\n#epsilon1 = data['ep1']\r\n#epsilon2 = data['ep2']\r\n#b = data['b']\r\n#d = data['d']\r\n#\r\n#X_list = []\r\n#U_list = []\r\n#\r\n#steps = len(tspan) # 500 snap shots\r\n#\r\n#x = X_reduced.flatten()[:,None]\r\n#y = Y_reduced.flatten()[:,None]\r\n#\r\n#for k in range(T1, T2):\r\n# t = tspan[k] * np.ones_like(x) \r\n# \r\n# u = usol[:,:,k].flatten()[:, None]\r\n# v = vsol[:,:,k].flatten()[:, None]\r\n# \r\n# X_list.append(np.concatenate([t, x, y], axis = 1))\r\n# U_list.append(np.concatenate([u, v], axis = 1))\r\n#\r\n#X_reduced = np.vstack(X_list)\r\n#U_reduced = np.vstack(U_list)\r\n#\r\n#data_dict = {'X_reduced': X_reduced, 'U_reduced': U_reduced, \r\n# 'tspan': tspan, 'T1': tspan[T1], 'T2': tspan[T2], \r\n# 'ep1':epsilon1, 'ep2':epsilon2, 'b':b, 'd':d}\r\n#\r\n#np.save('data_reduced.npy', data_dict)\r\n\r\n\r\n" }, { "path": "GrayScott2D/data/readme", "content": "\n" }, { "path": "GrayScott2D/models_tf.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Fri Sep 25 14:22:32 2020\r\n\r\n@author: Wsf12\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nimport numpy as np\r\nimport time\r\n\r\n\r\nclass Sampler:\r\n # Initialize the class\r\n def __init__(self, dim, coords, func, name=None):\r\n self.dim = dim\r\n self.coords = coords\r\n self.func = func\r\n self.name = name\r\n\r\n def sample(self, N):\r\n x = self.coords[0:1, :] + (self.coords[1:2, :] - self.coords[0:1, :]) * np.random.rand(N, self.dim)\r\n y = self.func(x)\r\n return x, y\r\n\r\n\r\nclass ResidualSampler:\r\n # Initialize the class\r\n def __init__(self, X, name=None):\r\n self.X = X\r\n self.N = self.X.shape[0]\r\n\r\n def sample(self, batch_size):\r\n idx = np.random.choice(self.N, batch_size, replace=False)\r\n X_batch = self.X[idx, :]\r\n return X_batch\r\n\r\n\r\nclass DataSampler:\r\n # Initialize the class\r\n def __init__(self, X, Y, name=None):\r\n self.X = X\r\n self.Y = Y\r\n self.N = self.X.shape[0]\r\n\r\n def sample(self, batch_size):\r\n idx = np.random.choice(self.N, batch_size, replace=False)\r\n X_batch = self.X[idx, :]\r\n Y_batch = self.Y[idx, :]\r\n return X_batch, Y_batch\r\n\r\n\r\nclass Gray_Scott2D:\r\n # Initialize the class\r\n def __init__(self, data_sampler, residual_sampler, layers, b, d):\r\n\r\n N = data_sampler.N\r\n X, U = data_sampler.sample(N)\r\n\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n self.mu_y, self.sigma_y = self.mu_X[2], self.sigma_X[2]\r\n\r\n self.mu_U, self.sigma_U = U.mean(0), U.std(0)\r\n self.mu_u, self.sigma_u = self.mu_U[0], self.sigma_U[0]\r\n self.mu_v, self.sigma_v = self.mu_U[1], self.sigma_U[1]\r\n\r\n # Samplers\r\n self.data_sampler = data_sampler\r\n self.residual_sampler = residual_sampler\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Parameters\r\n self.epsilon1 = tf.Variable(-10.0, dtype=tf.float32)\r\n self.epsilon2 = tf.Variable(-10.0, dtype=tf.float32)\r\n\r\n self.b = b\r\n self.d = d\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.v_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.w_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.u_pred, self.v_pred = self.net_u(self.t_u_tf,\r\n self.x_u_tf,\r\n self.y_u_tf)\r\n\r\n self.u_res_pred, self.v_res_pred = self.net_r(self.t_r_tf,\r\n self.x_r_tf,\r\n self.y_r_tf)\r\n\r\n # Data loss\r\n self.loss_u_data = tf.reduce_mean(tf.square(self.u_tf - self.u_pred))\r\n self.loss_v_data = tf.reduce_mean(tf.square(self.v_tf - self.v_pred))\r\n self.loss_data = self.loss_u_data + self.loss_v_data\r\n\r\n # Residual loss\r\n self.loss_res_u = tf.reduce_mean(tf.square(self.u_res_pred))\r\n self.loss_res_v = tf.reduce_mean(tf.square(self.v_res_pred))\r\n\r\n self.loss_res = self.loss_res_u + self.loss_res_v\r\n\r\n # Total loss\r\n self.loss = self.loss_data + self.loss_res\r\n\r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate,\r\n self.global_step,\r\n 1000, 0.9,\r\n staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss,\r\n global_step=self.global_step)\r\n\r\n # Logger\r\n self.loss_u_log = []\r\n self.loss_r_log = []\r\n\r\n self.ep1_log = []\r\n self.ep2_log = []\r\n\r\n self.saver = tf.train.Saver()\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 1):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.zeros([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n return weights, biases\r\n\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = np.sqrt(2 / (in_dim + out_dim))\r\n return tf.Variable(tf.truncated_normal([in_dim, out_dim], stddev=xavier_stddev), dtype=tf.float32)\r\n\r\n def neural_net(self, H):\r\n num_layers = len(self.layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n\r\n def net_u(self, t, x, y):\r\n # Compute scalar potentials\r\n out = self.neural_net(tf.concat([t, x, y], 1))\r\n u = out[:, 0:1]\r\n v = out[:, 1:2]\r\n\r\n # De-normalize\r\n u = u * self.sigma_u + self.mu_u\r\n v = v * self.sigma_v + self.mu_v\r\n\r\n return u, v\r\n\r\n def net_r(self, t, x, y):\r\n u, v = self.net_u(t, x, y)\r\n\r\n u_t = tf.gradients(u, t)[0] / self.sigma_t\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_y = tf.gradients(u, y)[0] / self.sigma_y\r\n\r\n v_t = tf.gradients(v, t)[0] / self.sigma_t\r\n v_x = tf.gradients(v, x)[0] / self.sigma_x\r\n v_y = tf.gradients(v, y)[0] / self.sigma_y\r\n\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n u_yy = tf.gradients(u_y, y)[0] / self.sigma_y\r\n\r\n v_xx = tf.gradients(v_x, x)[0] / self.sigma_x\r\n v_yy = tf.gradients(v_y, y)[0] / self.sigma_y\r\n\r\n u_res = u_t - tf.exp(self.epsilon1) * (u_xx + u_yy) - self.b * (1 - u) + u * tf.square(v)\r\n v_res = v_t - tf.exp(self.epsilon2) * (v_xx + v_yy) + self.d * v - u * tf.square(v)\r\n\r\n return u_res, v_res\r\n\r\n def fetch_minibatch_data(self, N):\r\n X, Y = self.data_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def fetch_minibatch_residual(self, N):\r\n X = self.residual_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n start_time = time.time()\r\n for it in range(nIter):\r\n X_u_batch, U_batch = self.fetch_minibatch_data(batch_size)\r\n X_r_batch, _ = self.fetch_minibatch_residual(batch_size)\r\n\r\n tf_dict = {self.t_u_tf: X_u_batch[:, 0:1], self.x_u_tf: X_u_batch[:, 1:2], self.y_u_tf: X_u_batch[:, 2:3],\r\n self.t_r_tf: X_r_batch[:, 0:1], self.x_r_tf: X_r_batch[:, 1:2], self.y_r_tf: X_r_batch[:, 2:3],\r\n self.u_tf: U_batch[:, 0:1], self.v_tf: U_batch[:, 1:2]}\r\n\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = time.time() - start_time\r\n loss_u_value = self.sess.run(self.loss_data, tf_dict)\r\n loss_r_value = self.sess.run(self.loss_res, tf_dict)\r\n\r\n ep1_value = self.sess.run(self.epsilon1)\r\n ep2_value = self.sess.run(self.epsilon2)\r\n\r\n self.loss_u_log.append(loss_u_value)\r\n self.loss_r_log.append(loss_r_value)\r\n\r\n self.ep1_log.append(np.exp(ep1_value))\r\n self.ep2_log.append(np.exp(ep2_value))\r\n\r\n print('It: %d, Data: %.3e, Residual: %.3e, Time: %.2f' %\r\n (it, loss_u_value, loss_r_value, elapsed))\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1_value), np.exp(ep2_value)))\r\n\r\n start_time = time.time()\r\n\r\n def predict(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1],\r\n self.x_u_tf: X_star[:, 1:2],\r\n self.y_u_tf: X_star[:, 2:3]}\r\n u_pred = self.sess.run(self.u_pred, tf_dict)\r\n v_pred = self.sess.run(self.v_pred, tf_dict)\r\n return u_pred, v_pred\r\n\r\n\r\nclass Gray_Scott2D_FF:\r\n # Initialize the class\r\n def __init__(self, data_sampler, residual_sampler, layers, b, d):\r\n\r\n N = data_sampler.N\r\n X, U = data_sampler.sample(N)\r\n\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n self.mu_y, self.sigma_y = self.mu_X[2], self.sigma_X[2]\r\n\r\n self.mu_U, self.sigma_U = U.mean(0), U.std(0)\r\n self.mu_u, self.sigma_u = self.mu_U[0], self.sigma_U[0]\r\n self.mu_v, self.sigma_v = self.mu_U[1], self.sigma_U[1]\r\n\r\n # Samplers\r\n self.data_sampler = data_sampler\r\n self.residual_sampler = residual_sampler\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # self.W_t = tf.Variable(tf.random_uniform([1, layers[0] // 2], minval=0, maxval=1), dtype=tf.float32)\r\n # self.b_t = tf.Variable(tf.random_uniform([1, layers[0]], dtype=tf.float32), dtype=tf.float32)\r\n #\r\n # self.W_x = tf.Variable(tf.random_uniform([2, layers[0] // 2], minval=0, maxval=20), dtype=tf.float32)\r\n # self.b_x = tf.Variable(tf.random_uniform([2, layers[0]], dtype=tf.float32), dtype=tf.float32)\r\n\r\n self.W_t = tf.Variable(tf.random_normal([1, layers[0] // 2], dtype=tf.float32) * 1, dtype=tf.float32,\r\n trainable=False)\r\n\r\n self.W_x = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 30, dtype=tf.float32,\r\n trainable=False)\r\n\r\n # Parameters\r\n self.epsilon1 = tf.Variable(-10.0, dtype=tf.float32)\r\n self.epsilon2 = tf.Variable(-10.0, dtype=tf.float32)\r\n\r\n self.b = b\r\n self.d = d\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.v_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.w_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.u_pred, self.v_pred = self.net_u(self.t_u_tf,\r\n self.x_u_tf,\r\n self.y_u_tf)\r\n\r\n self.u_res_pred, self.v_res_pred = self.net_r(self.t_r_tf,\r\n self.x_r_tf,\r\n self.y_r_tf)\r\n\r\n # Data loss\r\n self.loss_u_data = tf.reduce_mean(tf.square(self.u_tf - self.u_pred))\r\n self.loss_v_data = tf.reduce_mean(tf.square(self.v_tf - self.v_pred))\r\n self.loss_data = self.loss_u_data + self.loss_v_data\r\n\r\n # Residual loss\r\n self.loss_res_u = tf.reduce_mean(tf.square(self.u_res_pred))\r\n self.loss_res_v = tf.reduce_mean(tf.square(self.v_res_pred))\r\n\r\n self.loss_res = self.loss_res_u + self.loss_res_v\r\n\r\n # Total loss\r\n self.loss = self.loss_data + self.loss_res\r\n\r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate,\r\n self.global_step,\r\n 1000, 0.9,\r\n staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss,\r\n global_step=self.global_step)\r\n\r\n # Logger\r\n self.loss_u_log = []\r\n self.loss_r_log = []\r\n\r\n self.ep1_log = []\r\n self.ep2_log = []\r\n\r\n self.saver = tf.train.Saver()\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = np.sqrt(2 / (in_dim + out_dim))\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], stddev=xavier_stddev), dtype=tf.float32)\r\n\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n W = self.xavier_init(size=[layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n return weights, biases\r\n\r\n def neural_net(self, H):\r\n num_layers = len(self.layers)\r\n t = H[:, 0:1]\r\n x = H[:, 1:3]\r\n\r\n H_t = tf.concat([tf.sin(tf.matmul(t, self.W_t)),\r\n tf.cos(tf.matmul(t, self.W_t))], 1) # (N ,100))\r\n\r\n H_x = tf.concat([tf.sin(tf.matmul(x, self.W_x)),\r\n tf.cos(tf.matmul(x, self.W_x))], 1)\r\n\r\n for l in range(0, num_layers - 2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n\r\n H_t = tf.tanh(tf.add(tf.matmul(H_t, W), b))\r\n H_x = tf.tanh(tf.add(tf.matmul(H_x, W), b))\r\n\r\n H = tf.multiply(H_t, H_x)\r\n\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n\r\n def net_u(self, t, x, y):\r\n # Compute scalar potentials\r\n out = self.neural_net(tf.concat([t, x, y], 1))\r\n u = out[:, 0:1]\r\n v = out[:, 1:2]\r\n\r\n # De-normalize\r\n u = u * self.sigma_u + self.mu_u\r\n v = v * self.sigma_v + self.mu_v\r\n\r\n return u, v\r\n\r\n def net_r(self, t, x, y):\r\n u, v = self.net_u(t, x, y)\r\n\r\n u_t = tf.gradients(u, t)[0] / self.sigma_t\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_y = tf.gradients(u, y)[0] / self.sigma_y\r\n\r\n v_t = tf.gradients(v, t)[0] / self.sigma_t\r\n v_x = tf.gradients(v, x)[0] / self.sigma_x\r\n v_y = tf.gradients(v, y)[0] / self.sigma_y\r\n\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n u_yy = tf.gradients(u_y, y)[0] / self.sigma_y\r\n\r\n v_xx = tf.gradients(v_x, x)[0] / self.sigma_x\r\n v_yy = tf.gradients(v_y, y)[0] / self.sigma_y\r\n\r\n u_res = u_t - tf.exp(self.epsilon1) * (u_xx + u_yy) - self.b * (1 - u) + u * tf.square(v)\r\n v_res = v_t - tf.exp(self.epsilon2) * (v_xx + v_yy) + self.d * v - u * tf.square(v)\r\n\r\n # u_res = u_t - self.epsilon1 * (u_xx + u_yy) - self.b * (1 - u) + u * tf.square(v)\r\n # v_res = v_t - self.epsilon2 * (v_xx + v_yy) + self.d * v - u * tf.square(v)\r\n\r\n return u_res, v_res\r\n\r\n def fetch_minibatch_data(self, N):\r\n X, Y = self.data_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def fetch_minibatch_residual(self, N):\r\n X, Y = self.residual_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n start_time = time.time()\r\n for it in range(nIter):\r\n X_u_batch, U_batch = self.fetch_minibatch_data(batch_size)\r\n X_r_batch, _ = self.fetch_minibatch_residual(batch_size)\r\n\r\n tf_dict = {self.t_u_tf: X_u_batch[:, 0:1], self.x_u_tf: X_u_batch[:, 1:2], self.y_u_tf: X_u_batch[:, 2:3],\r\n self.t_r_tf: X_r_batch[:, 0:1], self.x_r_tf: X_r_batch[:, 1:2], self.y_r_tf: X_r_batch[:, 2:3],\r\n self.u_tf: U_batch[:, 0:1], self.v_tf: U_batch[:, 1:2]}\r\n\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 10 == 0:\r\n elapsed = time.time() - start_time\r\n loss_u_value = self.sess.run(self.loss_data, tf_dict)\r\n loss_r_value = self.sess.run(self.loss_res, tf_dict)\r\n\r\n ep1_value = self.sess.run(self.epsilon1)\r\n ep2_value = self.sess.run(self.epsilon2)\r\n\r\n self.loss_u_log.append(loss_u_value)\r\n self.loss_r_log.append(loss_r_value)\r\n\r\n self.ep1_log.append(np.exp(ep1_value))\r\n self.ep2_log.append(np.exp(ep2_value))\r\n\r\n print('It: %d, Data: %.3e, Residual: %.3e, Time: %.2f' %\r\n (it, loss_u_value, loss_r_value, elapsed))\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1_value), np.exp(ep2_value)))\r\n # print('ep1: {:.3e}, ep2: {:.3e}'.format(ep1_value, ep2_value))\r\n\r\n start_time = time.time()\r\n\r\n def predict(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1],\r\n self.x_u_tf: X_star[:, 1:2],\r\n self.y_u_tf: X_star[:, 2:3]}\r\n u_pred = self.sess.run(self.u_pred, tf_dict)\r\n v_pred = self.sess.run(self.v_pred, tf_dict)\r\n return u_pred, v_pred\r\n\r\n\r\nclass Gray_Scott2D_ST_mFF:\r\n # Initialize the class\r\n def __init__(self, data_sampler, residual_sampler, layers, b, d):\r\n\r\n N = data_sampler.N\r\n X, U = data_sampler.sample(N)\r\n\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n self.mu_y, self.sigma_y = self.mu_X[2], self.sigma_X[2]\r\n\r\n self.mu_U, self.sigma_U = U.mean(0), U.std(0)\r\n self.mu_u, self.sigma_u = self.mu_U[0], self.sigma_U[0]\r\n self.mu_v, self.sigma_v = self.mu_U[1], self.sigma_U[1]\r\n\r\n # Samplers\r\n self.data_sampler = data_sampler\r\n self.residual_sampler = residual_sampler\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n self.W_t = tf.Variable(tf.random_normal([1, layers[0] // 2], dtype=tf.float32) * 1, dtype=tf.float32,\r\n trainable=False)\r\n\r\n self.W1_x = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 1, dtype=tf.float32,\r\n trainable=False)\r\n self.W2_x = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 10, dtype=tf.float32,\r\n trainable=False)\r\n self.W3_x = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 50, dtype=tf.float32,\r\n trainable=False)\r\n\r\n # Parameters\r\n # self.epsilon1 = epsilon1\r\n # self.epsilon2 = epsilon2\r\n\r\n self.epsilon1 = tf.Variable(-10.0, dtype=tf.float32)\r\n self.epsilon2 = tf.Variable(-10.0, dtype=tf.float32)\r\n self.b = b\r\n self.d = d\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.v_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.w_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.y_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.u_pred, self.v_pred = self.net_u(self.t_u_tf,\r\n self.x_u_tf,\r\n self.y_u_tf)\r\n\r\n self.u_res_pred, self.v_res_pred = self.net_r(self.t_r_tf,\r\n self.x_r_tf,\r\n self.y_r_tf)\r\n\r\n # Data loss\r\n self.loss_u_data = tf.reduce_mean(tf.square(self.u_tf - self.u_pred))\r\n self.loss_v_data = tf.reduce_mean(tf.square(self.v_tf - self.v_pred))\r\n self.loss_data = self.loss_u_data + self.loss_v_data\r\n\r\n # Residual loss\r\n self.loss_res_u = tf.reduce_mean(tf.square(self.u_res_pred))\r\n self.loss_res_v = tf.reduce_mean(tf.square(self.v_res_pred))\r\n\r\n self.loss_res = self.loss_res_u + self.loss_res_v\r\n\r\n # Total loss\r\n self.loss = self.loss_data + self.loss_res\r\n\r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate,\r\n self.global_step,\r\n 5000, 0.9,\r\n staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss,\r\n global_step=self.global_step)\r\n\r\n # Logger\r\n self.loss_u_log = []\r\n self.loss_r_log = []\r\n\r\n self.ep1_log = []\r\n self.ep2_log = []\r\n\r\n self.saver = tf.train.Saver()\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = np.sqrt(2 / (in_dim + out_dim))\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], stddev=xavier_stddev), dtype=tf.float32)\r\n\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n W = self.xavier_init(size=[3 * layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n return weights, biases\r\n\r\n def neural_net(self, H):\r\n num_layers = len(self.layers)\r\n t = H[:, 0:1]\r\n x = H[:, 1:3]\r\n\r\n H_t = tf.concat([tf.sin(tf.matmul(t, self.W_t)),\r\n tf.cos(tf.matmul(t, self.W_t))], 1) # (N ,100))\r\n\r\n H1_x = tf.concat([tf.sin(tf.matmul(x, self.W1_x)),\r\n tf.cos(tf.matmul(x, self.W1_x))], 1)\r\n\r\n H2_x = tf.concat([tf.sin(tf.matmul(x, self.W2_x)),\r\n tf.cos(tf.matmul(x, self.W2_x))], 1)\r\n\r\n H3_x = tf.concat([tf.sin(tf.matmul(x, self.W3_x)),\r\n tf.cos(tf.matmul(x, self.W3_x))], 1)\r\n\r\n for l in range(0, num_layers - 2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n\r\n H_t = tf.tanh(tf.add(tf.matmul(H_t, W), b))\r\n\r\n H1_x = tf.tanh(tf.add(tf.matmul(H1_x, W), b))\r\n H2_x = tf.tanh(tf.add(tf.matmul(H2_x, W), b))\r\n H3_x = tf.tanh(tf.add(tf.matmul(H3_x, W), b))\r\n\r\n H1 = tf.multiply(H_t, H1_x)\r\n H2 = tf.multiply(H_t, H2_x)\r\n H3 = tf.multiply(H_t, H3_x)\r\n\r\n H = tf.concat([H1, H2, H3], 1)\r\n\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n\r\n def net_u(self, t, x, y):\r\n # Compute scalar potentials\r\n out = self.neural_net(tf.concat([t, x, y], 1))\r\n u = out[:, 0:1]\r\n v = out[:, 1:2]\r\n\r\n # De-normalize\r\n u = u * self.sigma_u + self.mu_u\r\n v = v * self.sigma_v + self.mu_v\r\n\r\n return u, v\r\n\r\n def net_r(self, t, x, y):\r\n u, v = self.net_u(t, x, y)\r\n\r\n u_t = tf.gradients(u, t)[0] / self.sigma_t\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_y = tf.gradients(u, y)[0] / self.sigma_y\r\n\r\n v_t = tf.gradients(v, t)[0] / self.sigma_t\r\n v_x = tf.gradients(v, x)[0] / self.sigma_x\r\n v_y = tf.gradients(v, y)[0] / self.sigma_y\r\n\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n u_yy = tf.gradients(u_y, y)[0] / self.sigma_y\r\n\r\n v_xx = tf.gradients(v_x, x)[0] / self.sigma_x\r\n v_yy = tf.gradients(v_y, y)[0] / self.sigma_y\r\n\r\n u_res = u_t - tf.exp(self.epsilon1) * (u_xx + u_yy) - self.b * (1 - u) + u * tf.square(v)\r\n v_res = v_t - tf.exp(self.epsilon2) * (v_xx + v_yy) + self.d * v - u * tf.square(v)\r\n\r\n return u_res, v_res\r\n\r\n def fetch_minibatch_data(self, N):\r\n X, Y = self.data_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def fetch_minibatch_residual(self, N):\r\n X, Y = self.residual_sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n start_time = time.time()\r\n for it in range(nIter):\r\n X_u_batch, U_batch = self.fetch_minibatch_data(batch_size)\r\n X_r_batch, _ = self.fetch_minibatch_residual(batch_size)\r\n\r\n tf_dict = {self.t_u_tf: X_u_batch[:, 0:1], self.x_u_tf: X_u_batch[:, 1:2], self.y_u_tf: X_u_batch[:, 2:3],\r\n self.t_r_tf: X_r_batch[:, 0:1], self.x_r_tf: X_r_batch[:, 1:2], self.y_r_tf: X_r_batch[:, 2:3],\r\n self.u_tf: U_batch[:, 0:1], self.v_tf: U_batch[:, 1:2]}\r\n\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = time.time() - start_time\r\n loss_u_value = self.sess.run(self.loss_data, tf_dict)\r\n loss_r_value = self.sess.run(self.loss_res, tf_dict)\r\n\r\n ep1_value = self.sess.run(self.epsilon1)\r\n ep2_value = self.sess.run(self.epsilon2)\r\n\r\n self.loss_u_log.append(loss_u_value)\r\n self.loss_r_log.append(loss_r_value)\r\n\r\n self.ep1_log.append(np.exp(ep1_value))\r\n self.ep2_log.append(np.exp(ep2_value))\r\n\r\n print('It: %d, Data: %.3e, Residual: %.3e, Time: %.2f' %\r\n (it, loss_u_value, loss_r_value, elapsed))\r\n\r\n print('ep1: {:.3e}, ep2: {:.3e}'.format(np.exp(ep1_value), np.exp(ep2_value)))\r\n\r\n start_time = time.time()\r\n\r\n def predict(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1],\r\n self.x_u_tf: X_star[:, 1:2],\r\n self.y_u_tf: X_star[:, 2:3]}\r\n u_pred = self.sess.run(self.u_pred, tf_dict)\r\n v_pred = self.sess.run(self.v_pred, tf_dict)\r\n return u_pred, v_pred\r\n\r\n\r\n" }, { "path": "Poisson1D/Compute_Jacobian.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Sat Jul 11 17:45:07 2020\r\n\r\n@author: sifan\r\n\"\"\"\r\n\r\nfrom __future__ import absolute_import\r\nfrom __future__ import division\r\nfrom __future__ import print_function\r\nimport tensorflow as tf\r\nfrom tensorflow.python.framework import ops\r\nfrom tensorflow.python.ops import array_ops\r\nfrom tensorflow.python.ops import check_ops\r\nfrom tensorflow.python.ops import gradients_impl as gradient_ops\r\nfrom tensorflow.python.ops.parallel_for import control_flow_ops\r\nfrom tensorflow.python.util import nest\r\n\r\ndef jacobian(output, inputs, use_pfor=True, parallel_iterations=None):\r\n \"\"\"Computes jacobian of `output` w.r.t. `inputs`.\r\n Args:\r\n output: A tensor.\r\n inputs: A tensor or a nested structure of tensor objects.\r\n use_pfor: If true, uses pfor for computing the jacobian. Else uses\r\n tf.while_loop.\r\n parallel_iterations: A knob to control how many iterations and dispatched in\r\n parallel. This knob can be used to control the total memory usage.\r\n Returns:\r\n A tensor or a nested structure of tensors with the same structure as\r\n `inputs`. Each entry is the jacobian of `output` w.r.t. to the corresponding\r\n value in `inputs`. If output has shape [y_1, ..., y_n] and inputs_i has\r\n shape [x_1, ..., x_m], the corresponding jacobian has shape\r\n [y_1, ..., y_n, x_1, ..., x_m]. Note that in cases where the gradient is\r\n sparse (IndexedSlices), jacobian function currently makes it dense and\r\n returns a Tensor instead. This may change in the future.\r\n \"\"\"\r\n flat_inputs = nest.flatten(inputs)\r\n output_tensor_shape = output.shape\r\n output_shape = array_ops.shape(output)\r\n output = array_ops.reshape(output, [-1])\r\n\r\n def loop_fn(i):\r\n y = array_ops.gather(output, i)\r\n return gradient_ops.gradients(y, flat_inputs, unconnected_gradients=tf.UnconnectedGradients.ZERO)\r\n\r\n try:\r\n output_size = int(output.shape[0])\r\n except TypeError:\r\n output_size = array_ops.shape(output)[0]\r\n\r\n if use_pfor:\r\n pfor_outputs = control_flow_ops.pfor(\r\n loop_fn, output_size, parallel_iterations=parallel_iterations)\r\n else:\r\n pfor_outputs = control_flow_ops.for_loop(\r\n loop_fn,\r\n [output.dtype] * len(flat_inputs),\r\n output_size,\r\n parallel_iterations=parallel_iterations)\r\n\r\n for i, out in enumerate(pfor_outputs):\r\n if isinstance(out, ops.Tensor):\r\n new_shape = array_ops.concat(\r\n [output_shape, array_ops.shape(out)[1:]], axis=0)\r\n out = array_ops.reshape(out, new_shape)\r\n out.set_shape(output_tensor_shape.concatenate(flat_inputs[i].shape))\r\n pfor_outputs[i] = out\r\n\r\n return nest.pack_sequence_as(inputs, pfor_outputs)" }, { "path": "Poisson1D/Poisson_1D.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Tue Sep 1 13:52:42 2020\r\n\r\n@author: Wsf12\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nfrom scipy.interpolate import griddata\r\nimport seaborn as sns\r\nfrom models_tf import Sampler, NN, NN_FF, NN_mFF\r\n\r\n\r\nif __name__ == '__main__':\r\n\r\n # Hyper-parameters\r\n a = 2\r\n b = 50\r\n\r\n # Exact solution\r\n def u(x, a, b):\r\n return np.sin(np.pi * a * x) + 0.1 * np.sin(np.pi * b * x)\r\n\r\n # Exact PDE residual\r\n def u_xx(x, a, b):\r\n return - (np.pi * a) ** 2 * np.sin(np.pi * a * x) - 0.1 * (np.pi * b) ** 2 * np.sin(np.pi * b * x)\r\n\r\n # Define computational domain\r\n bc1_coords = np.array([[0.0],\r\n [0.0]])\r\n\r\n bc2_coords = np.array([[1.0],\r\n [1.0]])\r\n\r\n dom_coords = np.array([[0.0],\r\n [1.0]])\r\n\r\n # Create boundary sampler\r\n bc1 = Sampler(1, bc1_coords, lambda x: u(x, a, b), name='Dirichlet BC1')\r\n bc2 = Sampler(1, bc2_coords, lambda x: u(x, a, b), name='Dirichlet BC2')\r\n\r\n bcs_samplers = [bc1, bc2]\r\n\r\n # Create residual sampler\r\n res_samplers = Sampler(1, dom_coords, lambda x: u_xx(x, a, b), name='Forcing')\r\n\r\n # Define model\r\n # For NN model, please use layers = [1, 100, 100, 1]\r\n layers = [100, 100, 1]\r\n \r\n # Hyper-parameter for Fourier features\r\n sigma = 10\r\n \r\n # NN: Vanilla MLP\r\n # NN_FF : Vanilla Fourier feature network\r\n # NN_mFF : Multi-scale Fourier feature network\r\n model = NN(layers, bcs_samplers, res_samplers, u, a, b, sigma)\r\n\r\n # Train model\r\n model.train(nIter=40000, batch_size=128, log_NTK=False, log_weights=False)\r\n\r\n # Create test data\r\n nn = 10000\r\n X_star = np.linspace(dom_coords[0, 0], dom_coords[1, 0], nn)[:, None]\r\n u_star = u(X_star, a, b)\r\n r_star = u_xx(X_star, a, b)\r\n\r\n # Predictions\r\n u_pred = model.predict_u(X_star)\r\n r_pred = model.predict_r(X_star)\r\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\r\n error_r = np.linalg.norm(r_star - r_pred, 2) / np.linalg.norm(r_star, 2)\r\n\r\n print('Relative L2 error_u: {:.2e}'.format(error_u))\r\n print('Relative L2 error_r: {:.2e}'.format(error_r))\r\n \r\n loss_bcs = model.loss_bcs_log\r\n loss_res = model.loss_res_log\r\n l2_error = model.l2_error_log\r\n \r\n # Plot\r\n fig = plt.figure(figsize=(18, 5))\r\n with sns.axes_style(\"darkgrid\"):\r\n plt.subplot(1, 3, 1)\r\n plt.plot(X_star, u_star, label='Exact')\r\n plt.plot(X_star, u_pred, '--', label='Predicted')\r\n plt.xlabel('$x$')\r\n plt.ylabel('$y$')\r\n plt.legend(fontsize=20, loc='upper left')\r\n plt.tight_layout()\r\n\r\n plt.subplot(1, 3, 2)\r\n plt.plot(X_star, u_star - u_pred, label='Error')\r\n plt.xlabel('$x$')\r\n plt.ylabel('Point-wise error')\r\n plt.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\r\n plt.tight_layout()\r\n\r\n plt.subplot(1, 3, 3)\r\n iters = 100 * np.arange(len(loss_res))\r\n\r\n plt.plot(iters, loss_res, label='$\\mathcal{L}_{r}$', linewidth=2)\r\n plt.plot(iters, loss_bcs, label='$\\mathcal{L}_{b}$', linewidth=2)\r\n plt.plot(iters, l2_error, label=r'$L^2$ error', linewidth=2)\r\n\r\n plt.yscale('log')\r\n plt.xlabel('iterations')\r\n plt.legend(loc='upper right', bbox_to_anchor=(1.0, 0.9), fontsize=20)\r\n plt.tight_layout()\r\n plt.show()\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n" }, { "path": "Poisson1D/models_tf.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Tue Sep 1 14:17:33 2020\r\n\r\n@author: Wsf12\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nfrom Compute_Jacobian import jacobian\r\nimport numpy as np\r\nimport timeit\r\n\r\nclass Sampler:\r\n # Initialize the class\r\n def __init__(self, dim, coords, func, name=None):\r\n self.dim = dim\r\n self.coords = coords\r\n self.func = func\r\n self.name = name\r\n\r\n def sample(self, N):\r\n x = self.coords[0:1, :] + (self.coords[1:2, :] - self.coords[0:1, :]) * np.random.rand(N, self.dim)\r\n y = self.func(x)\r\n return x, y\r\n\r\nclass NN:\r\n def __init__(self, layers, bcs_samplers, res_samplers, u, a, b, sigma):\r\n\r\n # Normalize the input\r\n X, _ = res_samplers.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_x, self.sigma_x = self.mu_X[0], self.sigma_X[0]\r\n \r\n # Samplers\r\n self.bcs_samplers = bcs_samplers\r\n self.res_samplers = res_samplers\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n \r\n # Define placeholders and computational graph\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n # Evaluate predictions\r\n self.u_bc1_pred = self.net_u(self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.x_bc2_tf)\r\n\r\n self.u_pred = self.net_u(self.x_u_tf)\r\n self.r_pred = self.net_r(self.x_r_tf)\r\n\r\n # Boundary loss\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred - self.u_bc1_tf))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred - self.u_bc2_tf))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n\r\n # Residual loss \r\n self.loss_res = tf.reduce_mean(tf.square(self.r_tf - self.r_pred))\r\n \r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs\r\n \r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n # Test data\r\n N_test = 1000\r\n \r\n self.X_star = np.linspace(0, 1, N_test)[:, None]\r\n self.u_star = u(self.X_star, a,b)\r\n\r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_res_log = []\r\n self.l2_error_log = []\r\n\r\n # Saver\r\n self.saver = tf.train.Saver()\r\n\r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n # Initialize network weights and biases using Xavier initialization\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 1):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n return weights, biases\r\n \r\n def forward_pass(self, H):\r\n num_layers = len(self.layers)\r\n \r\n for l in range(0, num_layers - 2): # number_layers - 1?\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n \r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n \r\n def net_u(self, x):\r\n u = self.forward_pass(x)\r\n return u\r\n\r\n # Forward pass for f\r\n def net_r(self, x):\r\n u = self.net_u(x)\r\n\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n\r\n res_u = u_xx\r\n return res_u\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n # Trains the model by minimizing the MSE loss\r\n def train(self, nIter=10000, batch_size=128, log_NTK=True, log_weights=True):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_samplers[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_samplers[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, f_batch = self.fetch_minibatch(self.res_samplers, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.x_bc1_tf: X_bc1_batch, self.x_bc2_tf: X_bc2_batch,\r\n self.u_bc1_tf: u_bc1_batch, self.u_bc2_tf: u_bc2_batch,\r\n self.x_u_tf: X_res_batch, self.x_r_tf: X_res_batch,\r\n self.r_tf: f_batch\r\n }\r\n \r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 1000 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_res_value = self.sess.run([self.loss_bcs, self.loss_res], tf_dict)\r\n \r\n u_pred = self.predict_u(self.X_star)\r\n error_u = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n \r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error_u)\r\n\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_res: %.3e ,Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_res_value, elapsed))\r\n\r\n start_time = timeit.default_timer()\r\n\r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_u_tf: X_star}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluates predictions at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_r_tf: X_star}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n\r\nclass NN_FF:\r\n def __init__(self, layers, bcs_samplers, res_samplers, u, a, b, sigma):\r\n\r\n # Normalize the input\r\n X, _ = res_samplers.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_x, self.sigma_x = self.mu_X[0], self.sigma_X[0]\r\n \r\n # Samplers\r\n self.bcs_samplers = bcs_samplers\r\n self.res_samplers = res_samplers\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Initialize Fourier features\r\n self.W = tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * sigma, dtype=tf.float32, trainable=False)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n \r\n # Define placeholders and computational graph\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n # Evaluate predictions\r\n self.u_bc1_pred = self.net_u(self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.x_bc2_tf)\r\n\r\n self.u_pred = self.net_u(self.x_u_tf)\r\n self.r_pred = self.net_r(self.x_r_tf)\r\n\r\n # Boundary loss\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred - self.u_bc1_tf))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred - self.u_bc2_tf))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n\r\n # Residual loss \r\n self.loss_res = tf.reduce_mean(tf.square(self.r_tf - self.r_pred))\r\n \r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs\r\n \r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 100, 0.99, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n \r\n # Test data\r\n N_test = 1000\r\n self.X_star = np.linspace(0, 1, N_test)[:, None]\r\n self.u_star = u(self.X_star, a, b)\r\n self.l2_error_log = []\r\n \r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_res_log = []\r\n self.l2_error_log = []\r\n\r\n # Saver\r\n self.saver = tf.train.Saver()\r\n\r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n # Initialize network weights and biases using Xavier initialization\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 1):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n return weights, biases\r\n \r\n def forward_pass(self, H):\r\n num_layers = len(self.layers)\r\n\r\n # Fourier feature encoding\r\n H = tf.concat([tf.sin(tf.matmul(H, self.W)),\r\n tf.cos(tf.matmul(H, self.W))], 1) \r\n\r\n for l in range(0, num_layers - 2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n \r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n \r\n def net_u(self, x):\r\n u = self.forward_pass(x)\r\n return u\r\n\r\n # Forward pass for f\r\n def net_r(self, x):\r\n u = self.net_u(x)\r\n\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n\r\n res_u = u_xx\r\n return res_u\r\n \r\n # Compute Jacobian for each weights and biases in each layer and retrun a list \r\n def compute_jacobian(self, f):\r\n J_list =[]\r\n \r\n L = len(self.weights) \r\n for i in range(L):\r\n J_w = jacobian(f, self.weights[i])\r\n J_list.append(J_w)\r\n \r\n for i in range(L):\r\n J_b = jacobian(f, self.biases[i])\r\n J_list.append(J_b)\r\n return J_list\r\n \r\n # Compute the empirical NTK = J J^T\r\n def compute_ntk(self, J1_list, x1, J2_list, x2):\r\n D = x1.shape[0]\r\n N = len(J1_list)\r\n \r\n Ker = tf.zeros((D,D))\r\n for k in range(N):\r\n J1 = tf.reshape(J1_list[k], shape=(D,-1))\r\n J2 = tf.reshape(J2_list[k], shape=(D,-1))\r\n \r\n K = tf.matmul(J1, tf.transpose(J2))\r\n Ker = Ker + K\r\n return Ker\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n # Trains the model by minimizing the MSE loss\r\n def train(self, nIter=10000, batch_size=128, log_NTK=True, log_weights=True):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_samplers[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_samplers[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, f_batch = self.fetch_minibatch(self.res_samplers, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.x_bc1_tf: X_bc1_batch, self.x_bc2_tf: X_bc2_batch,\r\n self.u_bc1_tf: u_bc1_batch, self.u_bc2_tf: u_bc2_batch,\r\n self.x_u_tf: X_res_batch, self.x_r_tf: X_res_batch,\r\n self.r_tf: f_batch\r\n }\r\n \r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_res_value = self.sess.run([self.loss_bcs, self.loss_res], tf_dict)\r\n \r\n u_pred = self.predict_u(self.X_star)\r\n error_u = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n \r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error_u)\r\n\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_res: %.3e ,Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_res_value, elapsed))\r\n\r\n start_time = timeit.default_timer()\r\n\r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_u_tf: X_star}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluates predictions at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_r_tf: X_star}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n \r\nclass NN_mFF:\r\n def __init__(self, layers, bcs_samplers, res_samplers, u,a, b, sigma):\r\n\r\n # Normalize the input\r\n X, _ = res_samplers.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_x, self.sigma_x = self.mu_X[0], self.sigma_X[0]\r\n \r\n # Samplers\r\n self.bcs_samplers = bcs_samplers\r\n self.res_samplers = res_samplers\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Initialize Fourier features\r\n self.W1 = tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * 1, dtype=tf.float32, trainable=False)\r\n self.W2 = tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * sigma, dtype=tf.float32, trainable=False)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n \r\n # Define placeholders and computational graph\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n # Evaluate predictions\r\n self.u_bc1_pred = self.net_u(self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.x_bc2_tf)\r\n\r\n self.u_pred = self.net_u(self.x_u_tf)\r\n self.r_pred = self.net_r(self.x_r_tf)\r\n\r\n # Boundary loss\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred - self.u_bc1_tf))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred - self.u_bc2_tf))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n\r\n # Residual loss \r\n self.loss_res = tf.reduce_mean(tf.square(self.r_tf - self.r_pred))\r\n \r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs\r\n \r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n \r\n # Test data\r\n N_test = 1000\r\n self.X_star = np.linspace(0, 1, N_test)[:, None]\r\n self.u_star = u(self.X_star, a,b)\r\n \r\n self.l2_error_log = []\r\n \r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_res_log = []\r\n self.saver = tf.train.Saver()\r\n\r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n # Initialize network weights and biases using Xavier initialization\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n \r\n num_layers = len(layers)\r\n \r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n W = self.xavier_init(size=[2 * layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n return weights, biases\r\n \r\n def forward_pass(self, H):\r\n num_layers = len(self.layers)\r\n\r\n # Fourier feature encodings\r\n H1 = tf.concat([tf.sin(tf.matmul(H, self.W1)),\r\n tf.cos(tf.matmul(H, self.W1))], 1)\r\n H2 = tf.concat([tf.sin(tf.matmul(H, self.W2)),\r\n tf.cos(tf.matmul(H, self.W2))], 1) # H1 (N ,50))\r\n\r\n for l in range(0, num_layers-2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H1 = tf.tanh(tf.add(tf.matmul(H1, W), b))\r\n \r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H2 = tf.tanh(tf.add(tf.matmul(H2, W), b))\r\n\r\n # Concatenate the network outputs\r\n H = tf.concat([H1, H2], 1)\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n \r\n def net_u(self, x):\r\n u = self.forward_pass(x)\r\n return u\r\n\r\n # Forward pass for f\r\n def net_r(self, x):\r\n u = self.net_u(x)\r\n\r\n u_x = tf.gradients(u, x)[0] / self.sigma_x\r\n u_xx = tf.gradients(u_x, x)[0] / self.sigma_x\r\n\r\n res_u = u_xx\r\n return res_u\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n # Trains the model by minimizing the MSE loss\r\n def train(self, nIter=10000, batch_size=128, log_NTK=True, log_weights=True):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_samplers[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_samplers[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, f_batch = self.fetch_minibatch(self.res_samplers, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.x_bc1_tf: X_bc1_batch, self.x_bc2_tf: X_bc2_batch,\r\n self.u_bc1_tf: u_bc1_batch, self.u_bc2_tf: u_bc2_batch,\r\n self.x_u_tf: X_res_batch, self.x_r_tf: X_res_batch,\r\n self.r_tf: f_batch\r\n }\r\n \r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_res_value = self.sess.run([self.loss_bcs, self.loss_res], tf_dict)\r\n\r\n u_pred = self.predict_u(self.X_star)\r\n error_u = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n \r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error_u)\r\n\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_res: %.3e ,Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_res_value, elapsed))\r\n\r\n start_time = timeit.default_timer()\r\n\r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_u_tf: X_star}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluates predictions at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_r_tf: X_star}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n\r\n" }, { "path": "README.md", "content": "## Multi-scale Fourier features for physics-informed neural networks\n\nCode and data (available upon request) accompanying the manuscript titled \"On the eigenvector bias of Fourier feature networks: From regression to solving multi-scale PDEs with physics-informed neural networks\", authored by Sifan Wang, Hanwen Wang, and Paris Perdikaris.\n\n## Abstract\n\nPhysics-informed neural networks (PINNs) are demonstrating remarkable promise in integrating physical models with gappy and noisy observational data, but they still struggle in cases where the target functions to be approximated exhibit high-frequency or multi-scale features. \nIn this work we investigate this limitation through the lens of Neural Tangent Kernel (NTK) theory and elucidate how PINNs are biased towards learning functions along the dominant eigen-directions of their limiting NTK. Using this observation, we construct novel architectures that employ spatio-temporal and multi-scale random Fourier features, and justify how such coordinate embedding layers can lead to robust and accurate PINN models. Numerical examples are presented for several challenging cases where conventional PINN models fail, including wave propagation and reaction-diffusion dynamics, illustrating how the proposed methods can be used to effectively tackle both forward and inverse problems involving partial differential equations with multi-scale behavior. \n\n## Citation\n\n @article{wang2021eigenvector,\n title={On the eigenvector bias of fourier feature networks: From regression to solving multi-scale pdes with physics-informed neural networks},\n author={Wang, Sifan and Wang, Hanwen and Perdikaris, Paris},\n journal={Computer Methods in Applied Mechanics and Engineering},\n volume={384},\n pages={113938},\n year={2021},\n publisher={Elsevier}\n }\n" }, { "path": "Regression/Compute_Jacobian.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Sat Jul 11 17:45:07 2020\r\n\r\n@author: sifan\r\n\"\"\"\r\n\r\nfrom __future__ import absolute_import\r\nfrom __future__ import division\r\nfrom __future__ import print_function\r\nimport tensorflow as tf\r\nfrom tensorflow.python.framework import ops\r\nfrom tensorflow.python.ops import array_ops\r\nfrom tensorflow.python.ops import check_ops\r\nfrom tensorflow.python.ops import gradients_impl as gradient_ops\r\nfrom tensorflow.python.ops.parallel_for import control_flow_ops\r\nfrom tensorflow.python.util import nest\r\n\r\ndef jacobian(output, inputs, use_pfor=True, parallel_iterations=None):\r\n \"\"\"Computes jacobian of `output` w.r.t. `inputs`.\r\n Args:\r\n output: A tensor.\r\n inputs: A tensor or a nested structure of tensor objects.\r\n use_pfor: If true, uses pfor for computing the jacobian. Else uses\r\n tf.while_loop.\r\n parallel_iterations: A knob to control how many iterations and dispatched in\r\n parallel. This knob can be used to control the total memory usage.\r\n Returns:\r\n A tensor or a nested structure of tensors with the same structure as\r\n `inputs`. Each entry is the jacobian of `output` w.r.t. to the corresponding\r\n value in `inputs`. If output has shape [y_1, ..., y_n] and inputs_i has\r\n shape [x_1, ..., x_m], the corresponding jacobian has shape\r\n [y_1, ..., y_n, x_1, ..., x_m]. Note that in cases where the gradient is\r\n sparse (IndexedSlices), jacobian function currently makes it dense and\r\n returns a Tensor instead. This may change in the future.\r\n \"\"\"\r\n flat_inputs = nest.flatten(inputs)\r\n output_tensor_shape = output.shape\r\n output_shape = array_ops.shape(output)\r\n output = array_ops.reshape(output, [-1])\r\n\r\n def loop_fn(i):\r\n y = array_ops.gather(output, i)\r\n return gradient_ops.gradients(y, flat_inputs, unconnected_gradients=tf.UnconnectedGradients.ZERO)\r\n\r\n try:\r\n output_size = int(output.shape[0])\r\n except TypeError:\r\n output_size = array_ops.shape(output)[0]\r\n\r\n if use_pfor:\r\n pfor_outputs = control_flow_ops.pfor(\r\n loop_fn, output_size, parallel_iterations=parallel_iterations)\r\n else:\r\n pfor_outputs = control_flow_ops.for_loop(\r\n loop_fn,\r\n [output.dtype] * len(flat_inputs),\r\n output_size,\r\n parallel_iterations=parallel_iterations)\r\n\r\n for i, out in enumerate(pfor_outputs):\r\n if isinstance(out, ops.Tensor):\r\n new_shape = array_ops.concat(\r\n [output_shape, array_ops.shape(out)[1:]], axis=0)\r\n out = array_ops.reshape(out, new_shape)\r\n out.set_shape(output_tensor_shape.concatenate(flat_inputs[i].shape))\r\n pfor_outputs[i] = out\r\n\r\n return nest.pack_sequence_as(inputs, pfor_outputs)" }, { "path": "Regression/models_tf.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Sat Jul 11 10:20:01 2020\r\n\r\n@author: sifan\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nfrom Compute_Jacobian import jacobian\r\nimport numpy as np\r\nimport timeit\r\n\r\n# Data Sampler\r\nclass Sampler:\r\n # Initialize the class\r\n def __init__(self, dim, coords, func, name=None):\r\n self.dim = dim\r\n self.coords = coords\r\n self.func = func\r\n self.name = name\r\n\r\n # Sample function\r\n def sample(self, N):\r\n x = self.coords[0:1, :] + (self.coords[1:2, :] - self.coords[0:1, :]) * np.random.rand(N, self.dim)\r\n y = self.func(x)\r\n return x, y\r\n \r\n\r\nclass NN_FF:\r\n def __init__(self, layers, X_u, Y_u, a, u, sigma):\r\n\r\n \"\"\"\r\n :param layers: Layers of the network\r\n :param X_u, Y_u: Training data\r\n :param a: Hyper-parameter of the target function\r\n :param u: the target function\r\n :param sigma: Hyper-parameter of the Fourier features\r\n \"\"\"\r\n\r\n self.mu_X, self.sigma_X = X_u.mean(0), X_u.std(0)\r\n self.mu_x, self.sigma_x = self.mu_X[0], self.sigma_X[0]\r\n\r\n # Normalize the input of the network\r\n self.X_u = (X_u - self.mu_X) / self.sigma_X\r\n self.Y_u = Y_u\r\n\r\n # Initialize Fourier features\r\n self.W = tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * sigma, dtype=tf.float32, trainable=False)\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n \r\n # Define the size of the Kernel\r\n self.D_u = X_u.shape[0]\r\n \r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_ntk_tf = tf.placeholder(tf.float32, shape=(self.D_u, 1))\r\n\r\n # Evaluate predictions\r\n self.u_pred = self.net_u(self.x_u_tf)\r\n\r\n # Evaluate NTK predictions\r\n self.u_ntk_pred = self.net_u(self.x_u_ntk_tf)\r\n \r\n # Boundary loss\r\n self.loss_u = tf.reduce_mean(tf.square(self.u_pred - self.u_tf)) \r\n \r\n # Total loss\r\n self.loss = self.loss_u\r\n\r\n # Define optimizer with learning rate schedule\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n # Model Saver\r\n self.saver = tf.train.Saver()\r\n\r\n # Compute the Jacobian for weights and biases in each hidden layer\r\n self.J_u = self.compute_jacobian(self.u_ntk_pred)\r\n\r\n # The empirical NTK = J J^T, compute NTK of PINNs\r\n self.K = self.compute_ntk(self.J_u, self.x_u_ntk_tf, self.J_u, self.x_u_ntk_tf)\r\n\r\n # Loss Logger\r\n self.loss_u_log = []\r\n\r\n # NTK logger\r\n self.K_log = []\r\n\r\n # Weights logger\r\n self.weights_log = []\r\n self.biases_log = []\r\n\r\n # Training error and test error\r\n N_train = 100\r\n N_test = 1000\r\n\r\n # Training data\r\n self.X_train = np.linspace(0, 1, N_train)[:, None]\r\n self.Y_train = u(self.X_train, a)\r\n\r\n # Test data\r\n self.X_test = np.linspace(0, 1, N_test)[:, None]\r\n self.Y_test = u(self.X_test, a)\r\n\r\n # Error loggers\r\n self.train_error_log = []\r\n self.test_error_log = []\r\n\r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n # NTK initialization\r\n def NTK_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n std = 1. / np.sqrt(in_dim)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * std,\r\n dtype=tf.float32)\r\n\r\n # Initialize network weights and biases using Xavier initialization\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 1):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n return weights, biases\r\n\r\n # Evaluate the forward pass\r\n def forward_pass(self, H):\r\n num_layers = len(self.layers)\r\n \r\n H = tf.concat([tf.sin(tf.matmul(H, self.W)),\r\n tf.cos(tf.matmul(H, self.W))], 1) \r\n\r\n for l in range(0, num_layers - 2): # number_layers - 1?\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n \r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n\r\n # Define the neural net\r\n def net_u(self, x):\r\n u = self.forward_pass(x)\r\n return u\r\n\r\n # Compute Jacobian for each weights and biases in each layer and retrun a list\r\n def compute_jacobian(self, f):\r\n J_list =[]\r\n L = len(self.weights)\r\n for i in range(L):\r\n J_w = jacobian(f, self.weights[i])\r\n J_list.append(J_w)\r\n\r\n for i in range(L):\r\n J_b = jacobian(f, self.biases[i])\r\n J_list.append(J_b)\r\n return J_list\r\n\r\n # Compute the empirical NTK = J J^T\r\n def compute_ntk(self, J1_list, x1, J2_list, x2):\r\n D1 = x1.shape[0]\r\n D2 = x2.shape[0]\r\n N = len(J1_list)\r\n\r\n Ker = tf.zeros((D1, D2))\r\n for k in range(N):\r\n J1 = tf.reshape(J1_list[k], shape=(D1, -1))\r\n J2 = tf.reshape(J2_list[k], shape=(D2, -1))\r\n\r\n K = tf.matmul(J1, tf.transpose(J2))\r\n Ker = Ker + K\r\n return Ker\r\n\r\n # Fetch minibatch\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n # Trains the model by minimizing the MSE loss\r\n def train(self, nIter=10000, log_NTK=True, log_weights=True):\r\n\r\n start_time = timeit.default_timer()\r\n\r\n for it in range(nIter):\r\n # Fetch mini-batches\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.x_u_tf: self.X_u, self.u_tf: self.Y_u\r\n }\r\n\r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_u_value = self.sess.run(self.loss_u, tf_dict)\r\n\r\n # Store the loss values\r\n self.loss_u_log.append(loss_u_value)\r\n\r\n # Compute the training error\r\n u_pred_train = self.predict_u(self.X_train)\r\n training_error = np.linalg.norm(self.Y_train - u_pred_train, 2) / np.linalg.norm(self.Y_train, 2)\r\n\r\n # Compute the test error\r\n u_pred_test = self.predict_u(self.X_test)\r\n test_error = np.linalg.norm(self.Y_test - u_pred_test, 2) / np.linalg.norm(self.Y_test, 2)\r\n\r\n # Store the training and test errors\r\n self.train_error_log.append(training_error)\r\n self.test_error_log.append(test_error)\r\n\r\n # print the loss values\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e,Time: %.2f' %\r\n (it, loss_value, loss_u_value, elapsed))\r\n\r\n start_time = timeit.default_timer()\r\n\r\n # Store the NTK matrix for every 100 iterations\r\n if log_NTK:\r\n # provide x, x' for NTK\r\n if it % 100 == 0:\r\n print(\"Compute NTK...\")\r\n tf_dict = {self.x_u_ntk_tf: self.X_u}\r\n K_value = self.sess.run(self.K, tf_dict)\r\n self.K_log.append(K_value)\r\n\r\n # Store the weights and biases of the network for every 100 iterations\r\n if log_weights:\r\n if it % 100 ==0:\r\n print(\"Weights stored...\")\r\n weights = self.sess.run(self.weights)\r\n biases = self.sess.run(self.biases)\r\n \r\n self.weights_log.append(weights)\r\n self.biases_log.append(biases)\r\n \r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.x_u_tf: X_star}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n\r\n" }, { "path": "Regression/regression.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Sat Jul 11 10:20:08 2020\r\n\r\n@author: sifan\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nimport seaborn as sns\r\nfrom models_tf import Sampler, NN_FF\r\n\r\n\r\nif __name__ == '__main__':\r\n\r\n # Define solution and its Laplace\r\n def u(x, a):\r\n return np.sin(np.pi * x) + np.cos(np.pi * a * x)\r\n\r\n # Define computational domain\r\n dom_coords = np.array([[0.0],\r\n [1.0]])\r\n\r\n # Training data on u(x) \r\n N_u = 100\r\n X_u = np.linspace(dom_coords[0, 0],\r\n dom_coords[1, 0], N_u)[:, None]\r\n\r\n a = 10 \r\n Y_u = u(X_u, a)\r\n \r\n # Test data\r\n nn = 1000\r\n X_star = np.linspace(dom_coords[0, 0], dom_coords[1, 0], nn)[:, None]\r\n u_star = u(X_star, a)\r\n \r\n # Define the model\r\n layers = [100, 100, 100, 1]\r\n sigma = 10 # Hyper-parameter of the Fourier features\r\n model = NN_FF(layers, X_u, Y_u, a, u, sigma)\r\n\r\n # Train the model for different epochs\r\n epoch_list = [10, 90, 900] # 1000 iterations in total\r\n u_pred_list = []\r\n\r\n for epoch in epoch_list:\r\n # Train the model\r\n model.train(nIter=epoch, log_NTK=True, log_weights=True)\r\n \r\n # Predictions\r\n u_pred = model.predict_u(X_star)\r\n u_pred_list.append(u_pred)\r\n\r\n # Evaulate the relative l2 error\r\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\r\n print('Relative L2 error_u: {:.2e}'.format(error_u))\r\n\r\n # Create loggers for the eigenvalues of the NTK\r\n lambda_K_log = []\r\n\r\n # Restore the NTK\r\n K_list = model.K_log\r\n\r\n for k in range(len(K_list)):\r\n K = K_list[k]\r\n\r\n # Compute eigenvalues\r\n lambda_K, eigvec_K = np.linalg.eig(K)\r\n \r\n # Sort in descresing order\r\n lambda_K = np.sort(np.real(lambda_K))[::-1]\r\n \r\n # Store eigenvalues\r\n lambda_K_log.append(lambda_K)\r\n \r\n # Change of the NTK\r\n kernel_diff_list = []\r\n K0 = K_list[0]\r\n for K in K_list:\r\n diff = np.linalg.norm(K - K0) / np.linalg.norm(K0) \r\n kernel_diff_list.append(diff)\r\n\r\n #######################\r\n #######################\r\n \r\n # Change of the weights\r\n def compute_weights_diff(weights_1, weights_2):\r\n weights = []\r\n N = len(weights_1)\r\n for k in range(N):\r\n weight = weights_1[k] - weights_2[k]\r\n weights.append(weight)\r\n return weights\r\n \r\n def compute_weights_norm(weights, biases):\r\n norm = 0\r\n for w in weights:\r\n norm = norm + np.sum(np.square(w))\r\n for b in biases:\r\n norm = norm + np.sum(np.square(b))\r\n norm = np.sqrt(norm)\r\n return norm\r\n \r\n # Restore the list weights and biases\r\n weights_log = model.weights_log\r\n biases_log = model.biases_log\r\n\r\n # The weights and biases at initialization\r\n weights_0 = weights_log[0]\r\n biases_0 = biases_log[0]\r\n \r\n weights_init_norm = compute_weights_norm(weights_0, biases_0)\r\n\r\n weights_change_list = []\r\n\r\n # Compute the change of weights and biases of the network\r\n N = len(weights_log)\r\n for k in range(N):\r\n weights_diff = compute_weights_diff(weights_log[k], weights_log[0])\r\n biases_diff = compute_weights_diff(biases_log[k], biases_log[0])\r\n \r\n weights_diff_norm = compute_weights_norm(weights_diff, biases_diff)\r\n weights_change = weights_diff_norm / weights_init_norm\r\n weights_change_list.append(weights_change)\r\n \r\n\r\n #################################\r\n ############## PLot##############\r\n #################################\r\n\r\n \r\n # Model predictions\r\n fig = plt.figure(1, figsize=(12, 5))\r\n plt.subplot(1,2,1)\r\n plt.plot(X_u, Y_u, 'o', label='Exact')\r\n plt.plot(X_star, u_pred, '--', label='u_pred')\r\n plt.legend()\r\n \r\n plt.subplot(1,2,2)\r\n plt.plot(X_star, u_star - u_pred, label='Error')\r\n plt.legend()\r\n plt.tight_layout()\r\n plt.show()\r\n \r\n # Eigenvalues of NTK\r\n fig = plt.figure(2, figsize=(6, 5))\r\n plt.plot(lambda_K_log[0], label = 'n=0')\r\n plt.plot(lambda_K_log[-1], '--', label = 'n=40,000')\r\n plt.xscale('log')\r\n plt.yscale('log')\r\n plt.xlabel('index')\r\n plt.ylabel(r'$\\lambda_{uu}$')\r\n plt.title(r'Eigenvalues of ${K}_{uu}$')\r\n plt.legend()\r\n plt.show()\r\n\r\n # Loss values\r\n loss_u = model.loss_u_log\r\n fig_3 = plt.figure(3, figsize=(6,5))\r\n plt.plot(loss_u, label='$\\mathcal{L}_{u_b}$')\r\n plt.yscale('log')\r\n plt.xlabel('iterations')\r\n plt.ylabel('Loss')\r\n plt.legend()\r\n plt.tight_layout()\r\n plt.show()\r\n \r\n\r\n\r\n # Visualize the eigenvectors of the NTK\r\n fig = plt.figure(figsize=(12, 6))\r\n with sns.axes_style(\"darkgrid\"):\r\n plt.subplot(2,3,1)\r\n plt.plot(X_u, np.real(eigvec_K[:,0]))\r\n plt.tight_layout()\r\n \r\n plt.subplot(2,3,2)\r\n plt.plot(X_u, np.real(eigvec_K[:,1]))\r\n plt.tight_layout()\r\n \r\n plt.subplot(2,3,3)\r\n plt.plot(X_u, np.real(eigvec_K[:,2]))\r\n plt.tight_layout()\r\n \r\n plt.subplot(2,3,4)\r\n plt.plot(X_u, np.real(eigvec_K[:,3]))\r\n plt.tight_layout()\r\n \r\n plt.subplot(2,3,5)\r\n plt.plot(X_u, np.real(eigvec_K[:,4]))\r\n plt.tight_layout()\r\n \r\n plt.subplot(2,3,6)\r\n plt.plot(X_u, np.real(eigvec_K[:,5]))\r\n \r\n plt.tight_layout()\r\n plt.show()\r\n \r\n # Visualize the eigenvalues of the NTK\r\n fig = plt.figure(figsize=(6, 5))\r\n with sns.axes_style(\"darkgrid\"):\r\n plt.plot(lambda_K_log[0], label=r'$\\sigma={}$'.format(sigma))\r\n plt.xscale('log')\r\n plt.yscale('log')\r\n plt.xlabel('index')\r\n plt.ylabel(r'$\\lambda$') \r\n plt.title('Spectrum')\r\n plt.tight_layout()\r\n plt.legend()\r\n plt.show()\r\n \r\n \r\n # Model predictions at different epoch\r\n fig = plt.figure(figsize=(12,4))\r\n with sns.axes_style(\"darkgrid\"):\r\n plt.subplot(1,3,1)\r\n plt.plot(X_u, Y_u, 'o')\r\n plt.plot(X_star, u_star, color = 'C0', alpha=0.4, linewidth=6)\r\n plt.plot(X_star, u_pred_list[0], color='C3', linestyle='--')\r\n plt.title('Epoch = 10')\r\n plt.tight_layout()\r\n \r\n plt.subplot(1,3,2)\r\n plt.plot(X_u, Y_u, 'o')\r\n plt.plot(X_star, u_star, color = 'C0', alpha=0.4, linewidth=6)\r\n plt.plot(X_star, u_pred_list[1], color='C3', linestyle='--')\r\n plt.title('Epoch = 100')\r\n plt.tight_layout()\r\n \r\n plt.subplot(1,3,3)\r\n plt.plot(X_u, Y_u, 'o')\r\n plt.plot(X_star, u_star, color = 'C0', alpha=0.4, linewidth=6)\r\n plt.plot(X_star, u_pred_list[2], color='C3', linestyle='--')\r\n plt.title('Epoch = 200')\r\n plt.tight_layout()\r\n plt.show()\r\n\r\n \r\n \r\n " }, { "path": "heat1D/heat1D.py", "content": "import tensorflow as tf\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nimport seaborn as sns\r\nfrom scipy.interpolate import griddata\r\nfrom models_tf import Sampler, heat1D_NN, heat1D_FF, heat1D_ST_FF\r\n\r\nif __name__ == '__main__':\r\n\r\n # Define exact solution\r\n def u(x, a, b):\r\n \"\"\"\r\n :param x: x = (t, x)\r\n \"\"\"\r\n\r\n t = x[:,0:1]\r\n x = x[:,1:2]\r\n \r\n return np.exp(-a * t) * np.sin(b * np.pi * x)\r\n\r\n def u_t(x, a, b):\r\n return - a * u(x, a, b)\r\n\r\n def u_xx(x, a, b):\r\n return - (b * np.pi)**2 * u(x, a, b)\r\n\r\n def f(x, a, b):\r\n k = a / (b * np.pi)**2 \r\n return u_t(x, a, b) - k * u_xx(x, a, b)\r\n\r\n # Define PDE residual\r\n def operator(u, t, x, k, sigma_t=1.0, sigma_x=1.0):\r\n u_t = tf.gradients(u, t)[0] / sigma_t\r\n u_x = tf.gradients(u, x)[0] / sigma_x\r\n u_xx = tf.gradients(u_x, x)[0] / sigma_x\r\n residual = u_t - k * u_xx\r\n return residual\r\n\r\n # Parameters of equations\r\n a = 1\r\n b = 500\r\n k = a / (b * np.pi)**2 \r\n\r\n # Domain boundaries\r\n ics_coords = np.array([[0.0, 0.0],\r\n [0.0, 1.0]])\r\n bc1_coords = np.array([[0.0, 0.0],\r\n [1.0, 0.0]])\r\n bc2_coords = np.array([[0.0, 1.0],\r\n [1.0, 1.0]])\r\n dom_coords = np.array([[0.0, 0.0],\r\n [1.0, 1.0]])\r\n\r\n # Create initial conditions samplers\r\n ics_sampler = Sampler(2, ics_coords, lambda x: u(x, a, b), name='Initial Condition 1')\r\n\r\n # Create boundary conditions samplers\r\n bc1 = Sampler(2, bc1_coords, lambda x: u(x, a, b), name='Dirichlet BC1')\r\n bc2 = Sampler(2, bc2_coords, lambda x: u(x, a, b), name='Dirichlet BC2')\r\n bcs_sampler = [bc1, bc2]\r\n\r\n # Create residual sampler\r\n res_sampler = Sampler(2, dom_coords, lambda x: f(x, a, b), name='Forcing')\r\n\r\n # Test data\r\n nn = 100 # nn = 1000\r\n t = np.linspace(dom_coords[0, 0], dom_coords[1, 0], nn)[:, None]\r\n x = np.linspace(dom_coords[0, 1], dom_coords[1, 1], nn)[:, None]\r\n t, x = np.meshgrid(t, x)\r\n X_star = np.hstack((t.flatten()[:, None], x.flatten()[:, None]))\r\n\r\n u_star = u(X_star, a, b)\r\n f_star = f(X_star, a, b)\r\n\r\n # Define model\r\n # heat1D_NN: Plain MLP\r\n # heat1D_FF: Plain Fourier feature network\r\n # heat1D_ST_FF: Spatial-temporal Plain Fourier feature network\r\n \r\n layers = [100, 100, 100, 1] # For heat1D_NN, use layers = [1, 100, 100, 100, 1]\r\n sigma = 500 # Hyper-parameter for Fourier feature embeddings\r\n model = heat1D_NN(layers, operator, k, \r\n ics_sampler, bcs_sampler, res_sampler, \r\n sigma, X_star, u_star)\r\n\r\n # Train model\r\n model.train(nIter=40000, batch_size=128)\r\n\r\n\r\n # Predictions\r\n u_pred = model.predict_u(X_star)\r\n\r\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\r\n print('Relative L2 error_u: {:.2e}'.format(error_u))\r\n \r\n\r\n # Grid data\r\n U_star = griddata(X_star, u_star.flatten(), (t, x), method='cubic')\r\n F_star = griddata(X_star, f_star.flatten(), (t, x), method='cubic')\r\n U_pred = griddata(X_star, u_pred.flatten(), (t, x), method='cubic')\r\n \r\n \r\n # Plot\r\n fig_1 = plt.figure(1, figsize=(18, 5))\r\n plt.subplot(1, 3, 1)\r\n plt.pcolor(t, x, U_star, cmap='jet')\r\n plt.colorbar()\r\n plt.xlabel('$t$')\r\n plt.ylabel('$x$')\r\n plt.title(r'Exact')\r\n plt.tight_layout()\r\n\r\n plt.subplot(1, 3, 2)\r\n plt.pcolor(t, x, U_pred, cmap='jet')\r\n plt.colorbar()\r\n plt.xlabel('$t$')\r\n plt.ylabel('$x$')\r\n plt.title(r'Predicted')\r\n plt.tight_layout()\r\n\r\n plt.subplot(1, 3, 3)\r\n plt.pcolor(t, x, np.abs(U_star - U_pred), cmap='jet')\r\n plt.colorbar()\r\n plt.xlabel('$t$')\r\n plt.ylabel('$x$')\r\n plt.title('Absolute error')\r\n plt.tight_layout()\r\n plt.show()\r\n\r\n loss_ics = model.loss_ics_log\r\n loss_bcs = model.loss_bcs_log\r\n loss_res = model.loss_res_log\r\n l2_error = model.l2_error_log\r\n \r\n fig_2 = plt.figure(2, figsize=(6, 5))\r\n with sns.axes_style(\"darkgrid\"):\r\n iters = 100 * np.arange(len(loss_res))\r\n \r\n plt.plot(iters, loss_res, label='$\\mathcal{L}_{r}$', linewidth=2)\r\n plt.plot(iters, loss_bcs, label='$\\mathcal{L}_{bc}$', linewidth=2)\r\n plt.plot(iters, loss_ics, label='$\\mathcal{L}_{ic}$', linewidth=2)\r\n plt.plot(iters, l2_error, label=r'$L^2$ error', linewidth=2)\r\n \r\n plt.yscale('log')\r\n plt.xlabel('iterations')\r\n plt.legend(ncol=2, fontsize=17)\r\n plt.tight_layout()\r\n plt.show()\r\n\r\n \r\n \r\n" }, { "path": "heat1D/models_tf.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Tue Sep 15 20:00:21 2020\r\n\r\n@author: Wsf12\r\n\"\"\"\r\n\r\nimport tensorflow as tf\r\nimport numpy as np\r\nimport timeit\r\n\r\n\r\nclass Sampler:\r\n # Initialize the class\r\n def __init__(self, dim, coords, func, name = None):\r\n self.dim = dim\r\n self.coords = coords\r\n self.func = func\r\n self.name = name\r\n def sample(self, N):\r\n x = self.coords[0:1,:] + (self.coords[1:2,:]-self.coords[0:1,:])*np.random.rand(N, self.dim)\r\n y = self.func(x)\r\n return x, y\r\n\r\n\r\nclass heat1D_NN:\r\n def __init__(self, layers, operator, k, ics_sampler, bcs_sampler, res_sampler, sigma, X_star, u_star):\r\n X, _ = res_sampler.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n\r\n # Samplers\r\n self.ics_sampler = ics_sampler\r\n self.bcs_sampler = bcs_sampler\r\n self.res_sampler = res_sampler\r\n\r\n # Define differential operator\r\n self.k = k\r\n self.operator = operator\r\n\r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n # Evaluate predictions\r\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\r\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\r\n\r\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\r\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\r\n\r\n # Boundary loss and Initial loss\r\n self.loss_ic = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n self.loss_ics = self.loss_ic\r\n\r\n # Residual loss\r\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\r\n\r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs + self.loss_ics\r\n\r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Test data\r\n self.X_star = X_star\r\n self.u_star = u_star\r\n\r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_ics_log = []\r\n self.loss_res_log = []\r\n self.saver = tf.train.Saver()\r\n\r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n\r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n\r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n W = self.xavier_init(size=[layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n\r\n return weights, biases\r\n\r\n # Evaluates the forward pass\r\n def forward_pass(self, H):\r\n num_layers = len(self.layers)\r\n\r\n for l in range(0, num_layers - 2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n\r\n # Forward pass for u\r\n def net_u(self, t, x):\r\n u = self.forward_pass(tf.concat([t, x], 1))\r\n return u\r\n\r\n # Forward pass for residual\r\n def net_r(self, t, x):\r\n u = self.net_u(t, x)\r\n residual = self.operator(u, t, x, self.k,\r\n self.sigma_t, self.sigma_x)\r\n return residual\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size)\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_sampler[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_sampler[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\r\n self.u_ics_tf: u_ics_batch,\r\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\r\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\r\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2]}\r\n\r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_ics_value, loss_res_value = self.sess.run([self.loss_bcs,\r\n self.loss_ics,\r\n self.loss_res], tf_dict)\r\n \r\n u_pred = self.predict_u(self.X_star)\r\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n\r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_ics_log.append(loss_ics_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error)\r\n\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_ics: %.3e, Loss_res: %.3e, Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_ics_value, loss_res_value, elapsed))\r\n\r\n start_time = timeit.default_timer()\r\n\r\n # Evaluate predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluate residual at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n\r\nclass heat1D_FF:\r\n def __init__(self, layers, operator, k, ics_sampler, bcs_sampler, res_sampler, sigma, X_star, u_star):\r\n X, _ = res_sampler.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n \r\n # Samplers\r\n self.ics_sampler = ics_sampler\r\n self.bcs_sampler = bcs_sampler\r\n self.res_sampler = res_sampler\r\n \r\n # Define differential operator\r\n self.k = k\r\n self.operator = operator\r\n \r\n # Fourier hyperparameter\r\n self.sigma = sigma\r\n\r\n self.W = tf.Variable(tf.random_normal([2, layers[0] //2], dtype=tf.float32) * sigma, dtype=tf.float32, trainable=False)\r\n \r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n \r\n # Evaluate predictions\r\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\r\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\r\n \r\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\r\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\r\n\r\n # Boundary loss and Initial loss\r\n self.loss_ic = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n self.loss_ics = self.loss_ic\r\n \r\n # Residual loss\r\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\r\n\r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs + self.loss_ics\r\n \r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Test data\r\n self.X_star = X_star\r\n self.u_star = u_star\r\n\r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_ics_log = []\r\n self.loss_res_log = []\r\n self.l2_error_log = []\r\n\r\n # Saver\r\n self.saver = tf.train.Saver()\r\n \r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n \r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n \r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n W = self.xavier_init(size=[layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n return weights, biases\r\n \r\n # Evaluates the forward pass\r\n def forward_pass(self, H):\r\n num_layers = len(self.layers) \r\n\r\n # Fourier feature encoding\r\n H = tf.concat([tf.sin(tf.matmul(H, self.W)),\r\n tf.cos(tf.matmul(H, self.W))], 1) \r\n\r\n # Pass through a MLP\r\n for l in range(0, num_layers-2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\r\n\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n \r\n # Forward pass for u\r\n def net_u(self, t, x):\r\n u = self.forward_pass(tf.concat([t, x], 1))\r\n return u\r\n \r\n # Forward pass for residual\r\n def net_r(self, t, x):\r\n u = self.net_u(t, x)\r\n residual = self.operator(u, t, x, self.k,\r\n self.sigma_t, self.sigma_x)\r\n return residual\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size)\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_sampler[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_sampler[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\r\n self.u_ics_tf: u_ics_batch,\r\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\r\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\r\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2]}\r\n\r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_ics_value, loss_res_value = self.sess.run([self.loss_bcs, \r\n self.loss_ics, \r\n self.loss_res], tf_dict)\r\n\r\n u_pred = self.predict_u(self.X_star)\r\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n\r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_ics_log.append(loss_ics_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error)\r\n\r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_ics: %.3e, Loss_res: %.3e, Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_ics_value, loss_res_value, elapsed))\r\n \r\n start_time = timeit.default_timer()\r\n\r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluates residual at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n\r\n\r\nclass heat1D_ST_FF:\r\n def __init__(self, layers, operator, k, ics_sampler, bcs_sampler, res_sampler, sigma, X_star, u_star):\r\n X, _ = res_sampler.sample(np.int32(1e5))\r\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\r\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\r\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\r\n \r\n # Samplers\r\n self.ics_sampler = ics_sampler\r\n self.bcs_sampler = bcs_sampler\r\n self.res_sampler = res_sampler\r\n \r\n # Define differential operator\r\n self.k = k\r\n self.operator = operator\r\n \r\n # Fourier hyperparameter\r\n self.sigma = sigma\r\n\r\n # Initialize spatial and temporal Fourier features\r\n self.W_t =tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * 1, dtype=tf.float32, trainable=False)\r\n self.W_x = tf.Variable(tf.random_normal([1, layers[0] //2], dtype=tf.float32) * sigma, dtype=tf.float32, trainable=False)\r\n \r\n # Initialize network weights and biases\r\n self.layers = layers\r\n self.weights, self.biases = self.initialize_NN(layers)\r\n\r\n # Define Tensorflow session\r\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\r\n\r\n # Define placeholders and computational graph\r\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.u_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n\r\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n self.r_tf = tf.placeholder(tf.float32, shape=(None, 1))\r\n \r\n # Evaluate predictions\r\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\r\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\r\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\r\n \r\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\r\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\r\n\r\n # Boundary loss and Initial loss\r\n self.loss_ic = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\r\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\r\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\r\n\r\n self.loss_bcs = self.loss_bc1 + self.loss_bc2\r\n self.loss_ics = self.loss_ic\r\n \r\n # Residual loss\r\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\r\n\r\n # Total loss\r\n self.loss = self.loss_res + self.loss_bcs + self.loss_ics\r\n \r\n self.global_step = tf.Variable(0, trainable=False)\r\n starter_learning_rate = 1e-3\r\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\r\n 1000, 0.9, staircase=False)\r\n # Passing global_step to minimize() will increment it at each step.\r\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\r\n\r\n # Test data\r\n self.X_star = X_star\r\n self.u_star = u_star\r\n\r\n # Logger\r\n self.loss_bcs_log = []\r\n self.loss_ics_log = []\r\n self.loss_res_log = []\r\n self.l2_error_log = []\r\n \r\n # Saver\r\n self.saver = tf.train.Saver()\r\n \r\n # Initialize Tensorflow variables\r\n init = tf.global_variables_initializer()\r\n self.sess.run(init)\r\n \r\n # Xavier initialization\r\n def xavier_init(self, size):\r\n in_dim = size[0]\r\n out_dim = size[1]\r\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\r\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\r\n dtype=tf.float32)\r\n\r\n # Initialize the network\r\n def initialize_NN(self, layers):\r\n weights = []\r\n biases = []\r\n \r\n num_layers = len(layers)\r\n for l in range(0, num_layers - 2):\r\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\r\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n# W = self.xavier_init(size=[2 *layers[-2], layers[-1]])\r\n W = self.xavier_init(size=[layers[-2], layers[-1]])\r\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\r\n weights.append(W)\r\n biases.append(b)\r\n \r\n return weights, biases\r\n \r\n # Evaluates the forward pass\r\n def forward_pass(self, H):\r\n num_layers = len(self.layers) \r\n t = H[:,0:1]\r\n x = H[:,1:2]\r\n\r\n # Temporal Fourier feature encoding\r\n H_t = tf.concat([tf.sin(tf.matmul(t, self.W_t)),\r\n tf.cos(tf.matmul(t, self.W_t))], 1) # H1 (N ,50))\r\n # Spatial Fourier feature encoding\r\n H_x = tf.concat([tf.sin(tf.matmul(x, self.W_x)),\r\n tf.cos(tf.matmul(x, self.W_x))], 1) \r\n\r\n # Pass through a MLP\r\n for l in range(0, num_layers-2):\r\n W = self.weights[l]\r\n b = self.biases[l]\r\n H_t = tf.tanh(tf.add(tf.matmul(H_t, W), b))\r\n H_x = tf.tanh(tf.add(tf.matmul(H_x, W), b))\r\n\r\n # Merge the outputs via point-wise multiplication\r\n H = tf.multiply(H_t, H_x) \r\n\r\n W = self.weights[-1]\r\n b = self.biases[-1]\r\n H = tf.add(tf.matmul(H, W), b)\r\n return H\r\n \r\n # Forward pass for u\r\n def net_u(self, t, x):\r\n u = self.forward_pass(tf.concat([t, x], 1))\r\n return u\r\n \r\n # Forward pass for residual\r\n def net_r(self, t, x):\r\n u = self.net_u(t, x)\r\n residual = self.operator(u, t, x, self.k,\r\n self.sigma_t, self.sigma_x)\r\n return residual\r\n\r\n def fetch_minibatch(self, sampler, N):\r\n X, Y = sampler.sample(N)\r\n X = (X - self.mu_X) / self.sigma_X\r\n return X, Y\r\n\r\n def train(self, nIter=10000, batch_size=128):\r\n\r\n start_time = timeit.default_timer()\r\n for it in range(nIter):\r\n # Fetch boundary mini-batches\r\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size)\r\n X_bc1_batch, u_bc1_batch = self.fetch_minibatch(self.bcs_sampler[0], batch_size)\r\n X_bc2_batch, u_bc2_batch = self.fetch_minibatch(self.bcs_sampler[1], batch_size)\r\n\r\n # Fetch residual mini-batch\r\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\r\n\r\n # Define a dictionary for associating placeholders with data\r\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\r\n self.u_ics_tf: u_ics_batch,\r\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\r\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\r\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2]}\r\n\r\n # Run the Tensorflow session to minimize the loss\r\n self.sess.run(self.train_op, tf_dict)\r\n\r\n # Print\r\n if it % 100 == 0:\r\n elapsed = timeit.default_timer() - start_time\r\n loss_value = self.sess.run(self.loss, tf_dict)\r\n loss_bcs_value, loss_ics_value, loss_res_value = self.sess.run([self.loss_bcs, \r\n self.loss_ics, \r\n self.loss_res], tf_dict)\r\n \r\n u_pred = self.predict_u(self.X_star)\r\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\r\n\r\n self.loss_bcs_log.append(loss_bcs_value)\r\n self.loss_ics_log.append(loss_ics_value)\r\n self.loss_res_log.append(loss_res_value)\r\n self.l2_error_log.append(error)\r\n \r\n print('It: %d, Loss: %.3e, Loss_bcs: %.3e, Loss_ics: %.3e, Loss_res: %.3e, Time: %.2f' %\r\n (it, loss_value, loss_bcs_value, loss_ics_value, loss_res_value, elapsed))\r\n \r\n start_time = timeit.default_timer()\r\n\r\n # Evaluates predictions at test points\r\n def predict_u(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\r\n u_star = self.sess.run(self.u_pred, tf_dict)\r\n return u_star\r\n\r\n # Evaluates PDE residual at test points\r\n def predict_r(self, X_star):\r\n X_star = (X_star - self.mu_X) / self.sigma_X\r\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\r\n r_star = self.sess.run(self.r_pred, tf_dict)\r\n return r_star\r\n\r\n\r\n \r\n" }, { "path": "wave1D/Compute_Jacobian.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nCreated on Sat Jul 11 17:45:07 2020\r\n\r\n@author: sifan\r\n\"\"\"\r\n\r\nfrom __future__ import absolute_import\r\nfrom __future__ import division\r\nfrom __future__ import print_function\r\nimport tensorflow as tf\r\nfrom tensorflow.python.framework import ops\r\nfrom tensorflow.python.ops import array_ops\r\nfrom tensorflow.python.ops import check_ops\r\nfrom tensorflow.python.ops import gradients_impl as gradient_ops\r\nfrom tensorflow.python.ops.parallel_for import control_flow_ops\r\nfrom tensorflow.python.util import nest\r\n\r\ndef jacobian(output, inputs, use_pfor=True, parallel_iterations=None):\r\n \"\"\"Computes jacobian of `output` w.r.t. `inputs`.\r\n Args:\r\n output: A tensor.\r\n inputs: A tensor or a nested structure of tensor objects.\r\n use_pfor: If true, uses pfor for computing the jacobian. Else uses\r\n tf.while_loop.\r\n parallel_iterations: A knob to control how many iterations and dispatched in\r\n parallel. This knob can be used to control the total memory usage.\r\n Returns:\r\n A tensor or a nested structure of tensors with the same structure as\r\n `inputs`. Each entry is the jacobian of `output` w.r.t. to the corresponding\r\n value in `inputs`. If output has shape [y_1, ..., y_n] and inputs_i has\r\n shape [x_1, ..., x_m], the corresponding jacobian has shape\r\n [y_1, ..., y_n, x_1, ..., x_m]. Note that in cases where the gradient is\r\n sparse (IndexedSlices), jacobian function currently makes it dense and\r\n returns a Tensor instead. This may change in the future.\r\n \"\"\"\r\n flat_inputs = nest.flatten(inputs)\r\n output_tensor_shape = output.shape\r\n output_shape = array_ops.shape(output)\r\n output = array_ops.reshape(output, [-1])\r\n\r\n def loop_fn(i):\r\n y = array_ops.gather(output, i)\r\n return gradient_ops.gradients(y, flat_inputs, unconnected_gradients=tf.UnconnectedGradients.ZERO)\r\n\r\n try:\r\n output_size = int(output.shape[0])\r\n except TypeError:\r\n output_size = array_ops.shape(output)[0]\r\n\r\n if use_pfor:\r\n pfor_outputs = control_flow_ops.pfor(\r\n loop_fn, output_size, parallel_iterations=parallel_iterations)\r\n else:\r\n pfor_outputs = control_flow_ops.for_loop(\r\n loop_fn,\r\n [output.dtype] * len(flat_inputs),\r\n output_size,\r\n parallel_iterations=parallel_iterations)\r\n\r\n for i, out in enumerate(pfor_outputs):\r\n if isinstance(out, ops.Tensor):\r\n new_shape = array_ops.concat(\r\n [output_shape, array_ops.shape(out)[1:]], axis=0)\r\n out = array_ops.reshape(out, new_shape)\r\n out.set_shape(output_tensor_shape.concatenate(flat_inputs[i].shape))\r\n pfor_outputs[i] = out\r\n\r\n return nest.pack_sequence_as(inputs, pfor_outputs)" }, { "path": "wave1D/wave1D.py", "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.interpolate import griddata\nfrom wave_models_tf import Sampler, Wave1D_NTK, Wave1D_NTK_mFF, Wave1D_NTK_ST_mFF\n\nif __name__ == '__main__':\n def u(x, a, c):\n \"\"\"\n :param x: x = (t, x)\n \"\"\"\n t = x[:,0:1]\n x = x[:,1:2]\n return np.sin(np.pi * x) * np.cos(c * np.pi * t) + \\\n np.sin(a * np.pi* x) * np.cos(a * c * np.pi * t)\n\n def f(x, a, c):\n N = x.shape[0]\n return np.zeros((N,1))\n\n def operator(u, t, x, c, sigma_t=1.0, sigma_x=1.0):\n u_t = tf.gradients(u, t)[0] / sigma_t\n u_x = tf.gradients(u, x)[0] / sigma_x\n u_tt = tf.gradients(u_t, t)[0] / sigma_t\n u_xx = tf.gradients(u_x, x)[0] / sigma_x\n residual = u_tt - c**2 * u_xx\n return residual\n \n # Hyper-parameters\n a = 2\n c = 10\n \n # Domain boundaries\n ics_coords = np.array([[0.0, 0.0],\n [0.0, 1.0]])\n bc1_coords = np.array([[0.0, 0.0],\n [1.0, 0.0]])\n bc2_coords = np.array([[0.0, 1.0],\n [1.0, 1.0]])\n dom_coords = np.array([[0.0, 0.0],\n [1.0, 1.0]])\n\n # Create initial conditions samplers\n ics_sampler = Sampler(2, ics_coords, lambda x: u(x, a, c), name='Initial Condition 1')\n\n # Create boundary conditions samplers\n bc1 = Sampler(2, bc1_coords, lambda x: u(x, a, c), name='Dirichlet BC1')\n bc2 = Sampler(2, bc2_coords, lambda x: u(x, a, c), name='Dirichlet BC2')\n bcs_sampler = [bc1, bc2]\n\n # Create residual sampler\n res_sampler = Sampler(2, dom_coords, lambda x: f(x, a, c), name='Forcing')\n \n # Test data\n nn = 200\n t = np.linspace(dom_coords[0, 0], dom_coords[1, 0], nn)[:, None]\n x = np.linspace(dom_coords[0, 1], dom_coords[1, 1], nn)[:, None]\n t, x = np.meshgrid(t, x)\n X_star = np.hstack((t.flatten()[:, None], x.flatten()[:, None]))\n\n u_star = u(X_star, a,c)\n\n # Define model\n # Wave1D_NTK: Plain MLP with NTK adaptive weights\n # Wave1D_NTK_mFF: Multi-scale Fourier feature network with NTK adaptive weights\n # Wave1D_NTK_ST_mFF: Spatial-temporal Fourier feature network with NTK adaptive weights\n \n# layers = [2, 200, 200, 200, 1] # if use Wave1D_NTK model\n layers = [200, 200, 200, 1]\n\n kernel_size = 120\n model = Wave1D_NTK_mFF(layers, operator, ics_sampler, bcs_sampler, res_sampler, c, kernel_size, X_star, u_star)\n \n \n # Train model\n itertaions = 40001\n model.train(nIter=itertaions, batch_size =kernel_size, log_NTK=True, update_weights=True)\n\n # Predictions\n u_pred = model.predict_u(X_star)\n f_pred = model.predict_r(X_star)\n\n error_u = np.linalg.norm(u_star - u_pred, 2) / np.linalg.norm(u_star, 2)\n\n print('Relative L2 error_u: %e' % (error_u))\n \n # Plot\n U_star = griddata(X_star, u_star.flatten(), (t, x), method='cubic')\n U_pred = griddata(X_star, u_pred.flatten(), (t, x), method='cubic')\n \n # Predictions \n fig = plt.figure(3, figsize=(18, 5))\n plt.subplot(1, 3, 1)\n plt.pcolor(t, x, U_star, cmap='jet')\n plt.colorbar()\n plt.xlabel('$t$')\n plt.ylabel('$x$')\n plt.title('Exact u(x)')\n\n plt.subplot(1, 3, 2)\n plt.pcolor(t, x, U_pred, cmap='jet')\n plt.colorbar()\n plt.xlabel('$t$')\n plt.ylabel('$x$')\n plt.title('Predicted u(x)')\n\n plt.subplot(1, 3, 3)\n plt.pcolor(t, x, np.abs(U_star - U_pred), cmap='jet')\n plt.colorbar()\n plt.xlabel('$t$')\n plt.ylabel('$x$')\n plt.title('Absolute error')\n plt.tight_layout()\n plt.show()\n \n # Restore loss_res and loss_bcs\n loss_res = model.loss_res_log\n loss_bcs = model.loss_bcs_log\n loss_u_t_ics = model.loss_ut_ics_log\n\n l2_error = model.l2_error_log\n\n fig = plt.figure(figsize=(6,5))\n iters =100 * np.arange(len(loss_res))\n with sns.axes_style(\"darkgrid\"):\n plt.plot(iters, loss_res, label='$\\mathcal{L}_{r}$')\n plt.plot(iters, loss_bcs, label='$\\mathcal{L}_{u}$')\n plt.plot(iters, loss_u_t_ics, label='$\\mathcal{L}_{u_t}$')\n plt.plot(iters, l2_error, label='$\\mathcal{L}^2 error$')\n plt.yscale('log')\n plt.xlabel('iterations')\n plt.legend(ncol=2)\n plt.tight_layout()\n plt.show()\n\n # NTK\n # Create loggers for eigenvalues of NTK\n lambda_K_u_log = []\n lambda_K_ut_log = []\n lambda_K_r_log = []\n \n # Restore the NTK\n K_u_list = model.K_u_log\n K_ut_list = model.K_ut_log\n K_r_list = model.K_r_log\n \n for k in range(len(K_u_list)):\n K_u = K_u_list[k]\n K_ut = K_ut_list[k]\n K_r = K_r_list[k]\n \n # Compute eigenvalues\n lambda_K_u, _ = np.linalg.eig(K_u)\n lambda_K_ut, _ = np.linalg.eig(K_ut)\n lambda_K_r, _ = np.linalg.eig(K_r)\n # Sort in descresing order\n lambda_K_u = np.sort(np.real(lambda_K_u))[::-1]\n lambda_K_ut = np.sort(np.real(lambda_K_ut))[::-1]\n lambda_K_r = np.sort(np.real(lambda_K_r))[::-1]\n \n # Store eigenvalues\n lambda_K_u_log.append(lambda_K_u)\n lambda_K_ut_log.append(lambda_K_ut)\n lambda_K_r_log.append(lambda_K_r)\n \n # Eigenvalues of NTK\n fig = plt.figure(figsize=(18, 5))\n plt.subplot(1,3,1)\n plt.plot(lambda_K_u_log[0], label = '$n=0$')\n plt.plot(lambda_K_u_log[1], '--', label = '$n=10,000$')\n plt.plot(lambda_K_u_log[4], '--', label = '$n=40,000$')\n plt.plot(lambda_K_u_log[-1], '--', label = '$n=80,000$')\n plt.xlabel('index')\n plt.xscale('log')\n plt.yscale('log')\n plt.legend()\n plt.title(r'Eigenvalues of ${K}_u$')\n\n plt.subplot(1,3,2)\n plt.plot(lambda_K_ut_log[0], label = '$n=0$')\n plt.plot(lambda_K_ut_log[1], '--',label = '$n=10,000$')\n plt.plot(lambda_K_ut_log[4], '--', label = '$n=40,000$')\n plt.plot(lambda_K_ut_log[-1], '--', label = '$n=80,000$')\n plt.xlabel('index')\n plt.xscale('log')\n plt.yscale('log')\n plt.legend()\n plt.title(r'Eigenvalues of ${K}_{u_t}$')\n \n ax =plt.subplot(1,3,3)\n plt.plot(lambda_K_r_log[0], label = '$n=0$')\n plt.plot(lambda_K_r_log[1], '--', label = '$n=10,000$')\n plt.plot(lambda_K_r_log[4], '--', label = '$n=40,000$')\n plt.plot(lambda_K_r_log[-1], '--', label = '$n=80,000$')\n plt.xscale('log')\n plt.yscale('log')\n plt.xlabel('index')\n plt.title(r'Eigenvalues of ${K}_{r}$')\n plt.legend()\n plt.tight_layout()\n plt.show()\n \n # Evolution of weights during training\n lambda_u_log = model.lambda_u_log\n lambda_ut_log = model.lambda_ut_log\n lambda_r_log = model.lambda_r_log \n\n fig = plt.figure(figsize=(6, 5))\n plt.plot(lambda_u_log, label='$\\lambda_u$')\n plt.plot(lambda_ut_log, label='$\\lambda_{u_t}$')\n plt.plot(lambda_r_log, label='$\\lambda_{r}$')\n plt.xlabel('iterations')\n plt.ylabel('$\\lambda$')\n plt.yscale('log')\n plt.legend( )\n plt.locator_params(axis='x',nbins=5)\n plt.tight_layout()\n plt.show() \n \n" }, { "path": "wave1D/wave_models_tf.py", "content": "import tensorflow as tf\nfrom Compute_Jacobian import jacobian\nimport numpy as np\nimport timeit\n\nclass Sampler:\n # Initialize the class\n def __init__(self, dim, coords, func, name = None):\n self.dim = dim\n self.coords = coords\n self.func = func\n self.name = name\n def sample(self, N):\n x = self.coords[0:1,:] + (self.coords[1:2,:]-self.coords[0:1,:])*np.random.rand(N, self.dim)\n y = self.func(x)\n return x, y\n\n\nclass Wave1D_NTK:\n # Plain MLP with NTK adaptive weights\n\n # Initialize the class\n def __init__(self, layers, operator, ics_sampler, bcs_sampler, res_sampler, c, kernel_size, X_star, u_star):\n\n # Normalize input\n X, _ = res_sampler.sample(np.int32(1e5))\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\n\n # Samplers\n self.operator = operator\n self.ics_sampler = ics_sampler\n self.bcs_sampler = bcs_sampler\n self.res_sampler = res_sampler\n\n # Test data\n self.X_star = X_star\n self.u_star = u_star\n\n # Initialize network weights and biases\n self.layers = layers\n self.weights, self.biases = self.initialize_NN(layers)\n \n # Initialize weights for losses\n self.lambda_u_val = np.array(1.0)\n self.lambda_ut_val = np.array(1.0)\n self.lambda_r_val = np.array(1.0)\n \n # Wave velocity\n self.c = tf.constant(c, dtype=tf.float32)\n\n # Size of the NTK\n self.kernel_size = kernel_size\n\n D1 = self.kernel_size # size of K_u\n D2 = self.kernel_size # size of K_ut\n D3 = self.kernel_size # size of K_r\n\n # Define Tensorflow session\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n\n # Define placeholders and computational graph\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.lambda_u_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_ut_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_r_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n\n self.t_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n self.x_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n \n self.t_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n self.x_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n \n self.t_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n self.x_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n\n # Evaluate predictions\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\n self.u_t_ics_pred = self.net_u_t(self.t_ics_tf, self.x_ics_tf)\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\n\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\n \n self.u_ntk_pred = self.net_u(self.t_u_ntk_tf, self.x_u_ntk_tf)\n self.ut_ntk_pred = self.net_u_t(self.t_ut_ntk_tf, self.x_ut_ntk_tf)\n self.r_ntk_pred = self.net_r(self.t_r_ntk_tf, self.x_r_ntk_tf)\n\n # Boundary loss and Initial loss\n self.loss_ics_u = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\n self.loss_ics_u_t = tf.reduce_mean(tf.square(self.u_t_ics_pred))\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\n\n self.loss_bcs = self.loss_ics_u + self.loss_bc1 + self.loss_bc2\n\n # Residual loss\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\n\n # Total loss\n self.loss = self.lambda_r_tf * self.loss_res + self.lambda_u_tf * self.loss_bcs + self.lambda_ut_tf * self.loss_ics_u_t \n\n # Define optimizer with learning rate schedule\n self.global_step = tf.Variable(0, trainable=False)\n starter_learning_rate = 1e-3\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\n 1000, 0.9, staircase=False)\n # Passing global_step to minimize() will increment it at each step.\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\n\n # Compute the Jacobian for weights and biases in each hidden layer \n self.J_u = self.compute_jacobian(self.u_ntk_pred)\n self.J_ut = self.compute_jacobian(self.ut_ntk_pred)\n self.J_r = self.compute_jacobian(self.r_ntk_pred)\n \n self.K_u = self.compute_ntk(self.J_u, D1, self.J_u, D1)\n self.K_ut = self.compute_ntk(self.J_ut, D2, self.J_ut, D2)\n self.K_r = self.compute_ntk(self.J_r, D3, self.J_r, D3)\n\n # Loss logger\n self.loss_bcs_log = []\n self.loss_ut_ics_log = []\n self.loss_res_log = []\n self.l2_error_log = []\n\n # NTK logger\n self.K_u_log = []\n self.K_ut_log = []\n self.K_r_log = []\n \n # weights logger\n self.lambda_u_log = []\n self.lambda_ut_log = []\n self.lambda_r_log = []\n \n # Initialize Tensorflow variables\n init = tf.global_variables_initializer()\n self.sess.run(init)\n\n # Saver\n self.saver = tf.train.Saver()\n\n # Initialize network weights and biases using Xavier initialization\n def initialize_NN(self, layers):\n # Xavier initialization\n def xavier_init(size):\n in_dim = size[0]\n out_dim = size[1]\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\n dtype=tf.float32)\n\n weights = []\n biases = []\n num_layers = len(layers)\n for l in range(0, num_layers - 1):\n W = xavier_init(size=[layers[l], layers[l + 1]])\n b = tf.Variable(tf.zeros([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\n weights.append(W)\n biases.append(b)\n return weights, biases\n\n # Evaluates the forward pass\n def forward_pass(self, H, layers, weights, biases):\n num_layers = len(layers)\n for l in range(0, num_layers - 2):\n W = weights[l]\n b = biases[l]\n H = tf.tanh(tf.add(tf.matmul(H, W), b))\n W = weights[-1]\n b = biases[-1]\n H = tf.add(tf.matmul(H, W), b)\n return H\n\n # Forward pass for u\n def net_u(self, t, x):\n u = self.forward_pass(tf.concat([t, x], 1),\n self.layers,\n self.weights,\n self.biases)\n return u\n\n def net_u_t(self, t, x):\n u_t = tf.gradients(self.net_u(t, x), t)[0] / self.sigma_t\n return u_t\n\n # Forward pass for f\n def net_r(self, t, x):\n u = self.net_u(t, x)\n residual = self.operator(u, t, x,\n self.c,\n self.sigma_t,\n self.sigma_x)\n return residual\n\n # Compute Jacobian for each weights and biases in each layer and retrun a list \n def compute_jacobian(self, f):\n J_list =[]\n L = len(self.weights) \n for i in range(L):\n J_w = jacobian(f, self.weights[i])\n J_list.append(J_w)\n \n for i in range(L):\n J_b = jacobian(f, self.biases[i])\n J_list.append(J_b)\n return J_list\n \n # Compute the empirical NTK = J J^T\n def compute_ntk(self, J1_list, D1, J2_list, D2):\n\n N = len(J1_list)\n \n Ker = tf.zeros((D1,D2))\n for k in range(N):\n J1 = tf.reshape(J1_list[k], shape=(D1,-1))\n J2 = tf.reshape(J2_list[k], shape=(D2,-1))\n \n K = tf.matmul(J1, tf.transpose(J2))\n Ker = Ker + K\n return Ker\n\n def fetch_minibatch(self, sampler, N):\n X, Y = sampler.sample(N)\n X = (X - self.mu_X) / self.sigma_X\n return X, Y\n\n # Trains the model by minimizing the MSE loss\n\n def train(self, nIter=10000, batch_size=128, log_NTK=False, update_weights=False):\n\n start_time = timeit.default_timer()\n for it in range(nIter):\n # Fetch boundary mini-batches\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size // 3)\n X_bc1_batch, _ = self.fetch_minibatch(self.bcs_sampler[0], batch_size // 3)\n X_bc2_batch, _ = self.fetch_minibatch(self.bcs_sampler[1], batch_size // 3)\n \n # Fetch residual mini-batch\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\n\n # Define a dictionary for associating placeholders with data\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\n self.u_ics_tf: u_ics_batch,\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2],\n self.lambda_u_tf: self.lambda_u_val,\n self.lambda_ut_tf: self.lambda_ut_val,\n self.lambda_r_tf: self.lambda_r_val}\n\n # Run the Tensorflow session to minimize the loss\n self.sess.run(self.train_op, tf_dict)\n\n # Print\n if it % 100 == 0:\n elapsed = timeit.default_timer() - start_time\n\n loss_value = self.sess.run(self.loss, tf_dict)\n loss_bcs_value = self.sess.run(self.loss_bcs, tf_dict)\n loss_ics_ut_value = self.sess.run(self.loss_ics_u_t, tf_dict)\n loss_res_value = self.sess.run(self.loss_res, tf_dict)\n\n u_pred = self.predict_u(self.X_star)\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\n\n self.loss_bcs_log.append(loss_bcs_value)\n self.loss_res_log.append(loss_res_value)\n self.loss_ut_ics_log.append(loss_ics_ut_value)\n self.l2_error_log.append(error)\n\n print('It: %d, Loss: %.3e, Loss_res: %.3e, Loss_bcs: %.3e, Loss_ut_ics: %.3e,, Time: %.2f' %\n (it, loss_value, loss_res_value, loss_bcs_value, loss_ics_ut_value, elapsed))\n\n print('lambda_u: {}'.format(self.lambda_u_val))\n print('lambda_ut: {}'.format(self.lambda_ut_val))\n print('lambda_r: {}'.format(self.lambda_r_val))\n\n start_time = timeit.default_timer()\n \n if log_NTK:\n X_bc_batch = np.vstack([X_ics_batch, X_bc1_batch, X_bc2_batch])\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size )\n \n if it % 1000 == 0:\n print(\"Compute NTK...\")\n tf_dict = {self.t_u_ntk_tf: X_bc_batch[:,0 :1], self.x_u_ntk_tf: X_bc_batch[:, 1:2],\n self.t_ut_ntk_tf: X_ics_batch[:, 0:1], self.x_ut_ntk_tf: X_ics_batch[:, 1:2],\n self.t_r_ntk_tf: X_res_batch[:, 0:1], self.x_r_ntk_tf: X_res_batch[:, 1:2]}\n\n # Compute NTK\n K_u_value, K_ut_value, K_r_value = self.sess.run([self.K_u, self.K_ut, self.K_r], tf_dict)\n \n lambda_K_sum = np.trace(K_u_value) + np.trace(K_ut_value) + \\\n np.trace(K_r_value)\n\n # Store NTK and weights\n self.K_u_log.append(K_u_value)\n self.K_ut_log.append(K_ut_value)\n self.K_r_log.append(K_r_value)\n\n if update_weights:\n self.lambda_u_val = lambda_K_sum / np.trace(K_u_value)\n self.lambda_ut_val = lambda_K_sum /np.trace(K_ut_value)\n self.lambda_r_val = lambda_K_sum / np.trace(K_r_value)\n\n # Store weights\n self.lambda_u_log.append(self.lambda_u_val)\n self.lambda_ut_log.append(self.lambda_ut_val)\n self.lambda_r_log.append(self.lambda_r_val)\n \n # Evaluates predictions at test points\n def predict_u(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\n u_star = self.sess.run(self.u_pred, tf_dict)\n return u_star\n\n # Evaluates predictions at test points\n def predict_r(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\n r_star = self.sess.run(self.r_pred, tf_dict)\n return r_star\n\n\nclass Wave1D_NTK_mFF:\n # Multiscale Fourier network with NTK adaptive weights\n\n # Initialize the class\n def __init__(self, layers, operator, ics_sampler, bcs_sampler, res_sampler, c, kernel_size, X_star, u_star):\n # Normalize input\n X, _ = res_sampler.sample(np.int32(1e5))\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\n\n # Samplers\n self.operator = operator\n self.ics_sampler = ics_sampler\n self.bcs_sampler = bcs_sampler\n self.res_sampler = res_sampler\n\n # Test data\n self.X_star = X_star\n self.u_star = u_star\n\n # Initialize multi-scale Fourier features\n self.W1 = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 1.0,\n dtype=tf.float32, trainable=True)\n\n self.W2 = tf.Variable(tf.random_normal([2, layers[0] // 2], dtype=tf.float32) * 10.0,\n dtype=tf.float32, trainable=True)\n\n # Initialize network weights and biases\n self.layers = layers\n self.weights, self.biases = self.initialize_NN(layers)\n\n # Initialize weights for losses\n self.lambda_u_val = np.array(1.0)\n self.lambda_ut_val = np.array(1.0)\n self.lambda_r_val = np.array(1.0)\n\n # Wave velocity constant\n self.c = tf.constant(c, dtype=tf.float32)\n\n # Size of the NTK\n self.kernel_size = kernel_size\n\n D1 = self.kernel_size # size of K_u\n D2 = self.kernel_size # size of K_ut\n D3 = self.kernel_size # size of K_r\n\n # Define Tensorflow session\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n\n # Define placeholders and computational graph\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.lambda_u_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_ut_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_r_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n\n self.t_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n self.x_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n\n self.t_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n self.x_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n\n self.t_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n self.x_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n\n # Evaluate predictions\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\n self.u_t_ics_pred = self.net_u_t(self.t_ics_tf, self.x_ics_tf)\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\n\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\n\n self.u_ntk_pred = self.net_u(self.t_u_ntk_tf, self.x_u_ntk_tf)\n self.ut_ntk_pred = self.net_u_t(self.t_ut_ntk_tf, self.x_ut_ntk_tf)\n self.r_ntk_pred = self.net_r(self.t_r_ntk_tf, self.x_r_ntk_tf)\n\n # Boundary loss and Initial loss\n self.loss_ics_u = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\n self.loss_ics_u_t = tf.reduce_mean(tf.square(self.u_t_ics_pred))\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\n\n self.loss_bcs = self.loss_ics_u + self.loss_bc1 + self.loss_bc2\n\n # Residual loss\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\n\n # Total loss\n self.loss = self.lambda_r_tf * self.loss_res + self.lambda_u_tf * self.loss_bcs + self.lambda_ut_tf * self.loss_ics_u_t\n\n # Define optimizer with learning rate schedule\n self.global_step = tf.Variable(0, trainable=False)\n starter_learning_rate = 1e-3\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\n 1000, 0.9, staircase=False)\n # Passing global_step to minimize() will increment it at each step.\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\n\n # Compute the Jacobian for weights and biases in each hidden layer\n self.J_u = self.compute_jacobian(self.u_ntk_pred)\n self.J_ut = self.compute_jacobian(self.ut_ntk_pred)\n self.J_r = self.compute_jacobian(self.r_ntk_pred)\n\n self.K_u = self.compute_ntk(self.J_u, D1, self.J_u, D1)\n self.K_ut = self.compute_ntk(self.J_ut, D2, self.J_ut, D2)\n self.K_r = self.compute_ntk(self.J_r, D3, self.J_r, D3)\n\n # Loss logger\n self.loss_bcs_log = []\n self.loss_ut_ics_log = []\n self.loss_res_log = []\n self.l2_error_log = []\n\n # NTK logger\n self.K_u_log = []\n self.K_ut_log = []\n self.K_r_log = []\n\n # weights logger\n self.lambda_u_log = []\n self.lambda_ut_log = []\n self.lambda_r_log = []\n\n # Initialize Tensorflow variables\n init = tf.global_variables_initializer()\n self.sess.run(init)\n\n # Saver\n self.saver = tf.train.Saver()\n\n # Initialize network weights and biases using Xavier initialization\n def xavier_init(self, size):\n in_dim = size[0]\n out_dim = size[1]\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\n dtype=tf.float32)\n\n def initialize_NN(self, layers):\n weights = []\n biases = []\n\n num_layers = len(layers)\n for l in range(0, num_layers - 2):\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\n weights.append(W)\n biases.append(b)\n\n W = self.xavier_init(size=[2 * layers[-2], layers[-1]])\n # W = self.xavier_init(size=[layers[-2], layers[-1]])\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\n weights.append(W)\n biases.append(b)\n\n return weights, biases\n\n # Evaluates the forward pass\n def forward_pass(self, H):\n num_layers = len(self.layers)\n\n # Multi-scale Fourier feature encodings\n H1 = tf.concat([tf.sin(tf.matmul(H, self.W1)),\n tf.cos(tf.matmul(H, self.W1))], 1)\n H2 = tf.concat([tf.sin(tf.matmul(H, self.W2)),\n tf.cos(tf.matmul(H, self.W2))], 1)\n\n for l in range(0, num_layers - 2):\n W = self.weights[l]\n b = self.biases[l]\n\n H1 = tf.tanh(tf.add(tf.matmul(H1, W), b))\n H2 = tf.tanh(tf.add(tf.matmul(H2, W), b))\n\n # Merge the outputs by concatenation\n H = tf.concat([H1, H2], 1)\n\n W = self.weights[-1]\n b = self.biases[-1]\n H = tf.add(tf.matmul(H, W), b)\n\n return H\n\n # Forward pass for u\n def net_u(self, t, x):\n u = self.forward_pass(tf.concat([t, x], 1))\n return u\n\n def net_u_t(self, t, x):\n u_t = tf.gradients(self.net_u(t, x), t)[0] / self.sigma_t\n return u_t\n\n # Forward pass for f\n def net_r(self, t, x):\n u = self.net_u(t, x)\n residual = self.operator(u, t, x,\n self.c,\n self.sigma_t,\n self.sigma_x)\n return residual\n\n # Compute Jacobian for each weights and biases in each layer and retrun a list\n def compute_jacobian(self, f):\n J_list = []\n L = len(self.weights)\n for i in range(L):\n J_w = jacobian(f, self.weights[i])\n J_list.append(J_w)\n\n for i in range(L):\n J_b = jacobian(f, self.biases[i])\n J_list.append(J_b)\n return J_list\n\n # Compute the empirical NTK = J J^T\n def compute_ntk(self, J1_list, D1, J2_list, D2):\n\n N = len(J1_list)\n\n Ker = tf.zeros((D1, D2))\n for k in range(N):\n J1 = tf.reshape(J1_list[k], shape=(D1, -1))\n J2 = tf.reshape(J2_list[k], shape=(D2, -1))\n\n K = tf.matmul(J1, tf.transpose(J2))\n Ker = Ker + K\n return Ker\n\n def fetch_minibatch(self, sampler, N):\n X, Y = sampler.sample(N)\n X = (X - self.mu_X) / self.sigma_X\n return X, Y\n\n # Trains the model by minimizing the MSE loss\n def train(self, nIter=10000, batch_size=128, log_NTK=False, update_weights=False):\n\n start_time = timeit.default_timer()\n for it in range(nIter):\n # Fetch boundary mini-batches\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size // 3)\n X_bc1_batch, _ = self.fetch_minibatch(self.bcs_sampler[0], batch_size // 3)\n X_bc2_batch, _ = self.fetch_minibatch(self.bcs_sampler[1], batch_size // 3)\n\n # Fetch residual mini-batch\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\n\n # Define a dictionary for associating placeholders with data\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\n self.u_ics_tf: u_ics_batch,\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2],\n self.lambda_u_tf: self.lambda_u_val,\n self.lambda_ut_tf: self.lambda_ut_val,\n self.lambda_r_tf: self.lambda_r_val}\n\n # Run the Tensorflow session to minimize the loss\n self.sess.run(self.train_op, tf_dict)\n\n # Print\n if it % 100 == 0:\n elapsed = timeit.default_timer() - start_time\n\n loss_value = self.sess.run(self.loss, tf_dict)\n loss_bcs_value = self.sess.run(self.loss_bcs, tf_dict)\n loss_ics_ut_value = self.sess.run(self.loss_ics_u_t, tf_dict)\n loss_res_value = self.sess.run(self.loss_res, tf_dict)\n\n u_pred = self.predict_u(self.X_star)\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\n\n self.loss_bcs_log.append(loss_bcs_value)\n self.loss_res_log.append(loss_res_value)\n self.loss_ut_ics_log.append(loss_ics_ut_value)\n self.l2_error_log.append(error)\n\n print('It: %d, Loss: %.3e, Loss_res: %.3e, Loss_bcs: %.3e, Loss_ut_ics: %.3e,, Time: %.2f' %\n (it, loss_value, loss_res_value, loss_bcs_value, loss_ics_ut_value, elapsed))\n\n print('lambda_u: {}'.format(self.lambda_u_val))\n print('lambda_ut: {}'.format(self.lambda_ut_val))\n print('lambda_r: {}'.format(self.lambda_r_val))\n\n start_time = timeit.default_timer()\n\n if log_NTK:\n X_bc_batch = np.vstack([X_ics_batch, X_bc1_batch, X_bc2_batch])\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size)\n\n if it % 100 == 0:\n print(\"Compute NTK...\")\n tf_dict = {self.t_u_ntk_tf: X_bc_batch[:, 0:1], self.x_u_ntk_tf: X_bc_batch[:, 1:2],\n self.t_ut_ntk_tf: X_ics_batch[:, 0:1], self.x_ut_ntk_tf: X_ics_batch[:, 1:2],\n self.t_r_ntk_tf: X_res_batch[:, 0:1], self.x_r_ntk_tf: X_res_batch[:, 1:2]}\n\n K_u_value, K_ut_value, K_r_value = self.sess.run([self.K_u, self.K_ut, self.K_r], tf_dict)\n\n # Store NTK\n self.K_u_log.append(K_u_value)\n self.K_ut_log.append(K_ut_value)\n self.K_r_log.append(K_r_value)\n\n if update_weights:\n lambda_K_sum = np.trace(K_u_value) + np.trace(K_ut_value) + \\\n np.trace(K_r_value)\n\n # Update weights\n self.lambda_u_val = lambda_K_sum / np.trace(K_u_value)\n self.lambda_ut_val = lambda_K_sum / np.trace(K_ut_value)\n self.lambda_r_val = lambda_K_sum / np.trace(K_r_value)\n\n # Store weights\n self.lambda_u_log.append(self.lambda_u_val)\n self.lambda_ut_log.append(self.lambda_ut_val)\n self.lambda_r_log.append(self.lambda_r_val)\n\n # Evaluates predictions at test points\n def predict_u(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\n u_star = self.sess.run(self.u_pred, tf_dict)\n return u_star\n\n # Evaluates predictions at test points\n def predict_r(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\n r_star = self.sess.run(self.r_pred, tf_dict)\n return r_star\n\n\nclass Wave1D_NTK_ST_mFF:\n # Initialize the class\n def __init__(self, layers, operator, ics_sampler, bcs_sampler, res_sampler, c, kernel_size, X_star, u_star):\n # Normalization constants\n X, _ = res_sampler.sample(np.int32(1e5))\n self.mu_X, self.sigma_X = X.mean(0), X.std(0)\n self.mu_t, self.sigma_t = self.mu_X[0], self.sigma_X[0]\n self.mu_x, self.sigma_x = self.mu_X[1], self.sigma_X[1]\n\n # Samplers\n self.operator = operator\n self.ics_sampler = ics_sampler\n self.bcs_sampler = bcs_sampler\n self.res_sampler = res_sampler\n\n # Test data\n self.X_star = X_star\n self.u_star = u_star\n\n # Initialize spatial and temporal Fourier features\n self.W1_t = tf.Variable(tf.random_normal([1, layers[0] // 2], dtype=tf.float32) * 1.0,\n dtype=tf.float32, trainable=False)\n\n self.W2_t = tf.Variable(tf.random_normal([1, layers[0] // 2], dtype=tf.float32) * 10.0,\n dtype=tf.float32, trainable=False)\n\n self.W1_x = tf.Variable(tf.random_normal([1, layers[0] // 2], dtype=tf.float32) * 1.0,\n dtype=tf.float32, trainable=False)\n\n # Initialize network weights and biases\n self.layers = layers\n self.weights, self.biases = self.initialize_NN(layers)\n\n # weights\n self.lambda_u_val = np.array(1.0)\n self.lambda_ut_val = np.array(1.0)\n self.lambda_r_val = np.array(1.0)\n\n # Wave velocity constant\n self.c = tf.constant(c, dtype=tf.float32)\n\n # Size of NTK\n self.kernel_size = kernel_size\n\n D1 = self.kernel_size # size of K_u\n D2 = self.kernel_size # size of K_ut\n D3 = self.kernel_size # size of K_r\n\n # Define Tensorflow session\n self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n\n # Define placeholders and computational graph\n self.t_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_u_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.u_ics_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc1_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_bc2_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.t_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n self.x_r_tf = tf.placeholder(tf.float32, shape=(None, 1))\n\n self.lambda_u_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_ut_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n self.lambda_r_tf = tf.placeholder(tf.float32, shape=self.lambda_u_val.shape)\n\n self.t_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n self.x_u_ntk_tf = tf.placeholder(tf.float32, shape=(D1, 1))\n\n self.t_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n self.x_ut_ntk_tf = tf.placeholder(tf.float32, shape=(D2, 1))\n\n self.t_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n self.x_r_ntk_tf = tf.placeholder(tf.float32, shape=(D3, 1))\n\n # Evaluate predictions\n self.u_ics_pred = self.net_u(self.t_ics_tf, self.x_ics_tf)\n self.u_t_ics_pred = self.net_u_t(self.t_ics_tf, self.x_ics_tf)\n self.u_bc1_pred = self.net_u(self.t_bc1_tf, self.x_bc1_tf)\n self.u_bc2_pred = self.net_u(self.t_bc2_tf, self.x_bc2_tf)\n\n self.u_pred = self.net_u(self.t_u_tf, self.x_u_tf)\n self.r_pred = self.net_r(self.t_r_tf, self.x_r_tf)\n\n self.u_ntk_pred = self.net_u(self.t_u_ntk_tf, self.x_u_ntk_tf)\n self.ut_ntk_pred = self.net_u_t(self.t_ut_ntk_tf, self.x_ut_ntk_tf)\n self.r_ntk_pred = self.net_r(self.t_r_ntk_tf, self.x_r_ntk_tf)\n\n # Boundary loss and Initial loss\n self.loss_ics_u = tf.reduce_mean(tf.square(self.u_ics_tf - self.u_ics_pred))\n self.loss_ics_u_t = tf.reduce_mean(tf.square(self.u_t_ics_pred))\n self.loss_bc1 = tf.reduce_mean(tf.square(self.u_bc1_pred))\n self.loss_bc2 = tf.reduce_mean(tf.square(self.u_bc2_pred))\n\n self.loss_bcs = self.loss_ics_u + self.loss_bc1 + self.loss_bc2\n\n # Residual loss\n self.loss_res = tf.reduce_mean(tf.square(self.r_pred))\n\n # Total loss\n self.loss = self.lambda_r_tf * self.loss_res + self.lambda_u_tf * self.loss_bcs + self.lambda_ut_tf * self.loss_ics_u_t\n\n # Define optimizer with learning rate schedule\n self.global_step = tf.Variable(0, trainable=False)\n starter_learning_rate = 1e-3\n self.learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step,\n 1000, 0.9, staircase=False)\n # Passing global_step to minimize() will increment it at each step.\n self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)\n\n # Compute the Jacobian for weights and biases in each hidden layer\n self.J_u = self.compute_jacobian(self.u_ntk_pred)\n self.J_ut = self.compute_jacobian(self.ut_ntk_pred)\n self.J_r = self.compute_jacobian(self.r_ntk_pred)\n\n self.K_u = self.compute_ntk(self.J_u, D1, self.J_u, D1)\n self.K_ut = self.compute_ntk(self.J_ut, D2, self.J_ut, D2)\n self.K_r = self.compute_ntk(self.J_r, D3, self.J_r, D3)\n\n # Loss logger\n self.loss_bcs_log = []\n self.loss_ut_ics_log = []\n self.loss_res_log = []\n self.l2_error_log = []\n\n # NTK logger\n self.K_u_log = []\n self.K_ut_log = []\n self.K_r_log = []\n\n # weights logger\n self.lambda_u_log = []\n self.lambda_ut_log = []\n self.lambda_r_log = []\n\n # Initialize Tensorflow variables\n init = tf.global_variables_initializer()\n self.sess.run(init)\n\n # Saver\n self.saver = tf.train.Saver()\n\n # Initialize network weights and biases using Xavier initialization\n def xavier_init(self, size):\n in_dim = size[0]\n out_dim = size[1]\n xavier_stddev = 1. / np.sqrt((in_dim + out_dim) / 2.)\n return tf.Variable(tf.random_normal([in_dim, out_dim], dtype=tf.float32) * xavier_stddev,\n dtype=tf.float32)\n\n def initialize_NN(self, layers):\n weights = []\n biases = []\n\n num_layers = len(layers)\n for l in range(0, num_layers - 2):\n W = self.xavier_init(size=[layers[l], layers[l + 1]])\n b = tf.Variable(tf.random_normal([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)\n weights.append(W)\n biases.append(b)\n\n W = self.xavier_init(size=[2 * layers[-2], layers[-1]])\n b = tf.Variable(tf.random_normal([1, layers[-1]], dtype=tf.float32), dtype=tf.float32)\n weights.append(W)\n biases.append(b)\n\n return weights, biases\n\n # Evaluates the forward pass\n def forward_pass(self, H):\n num_layers = len(self.layers)\n\n t = H[:, 0:1]\n x = H[:, 1:2]\n\n # Temporal and spatial Fourier feature encodings\n H1_t = tf.concat([tf.sin(tf.matmul(t, self.W1_t)),\n tf.cos(tf.matmul(t, self.W1_t))], 1)\n\n H2_t = tf.concat([tf.sin(tf.matmul(t, self.W2_t)),\n tf.cos(tf.matmul(t, self.W2_t))], 1)\n\n H1_x = tf.concat([tf.sin(tf.matmul(x, self.W1_x)),\n tf.cos(tf.matmul(x, self.W1_x))], 1)\n\n for l in range(0, num_layers - 2):\n W = self.weights[l]\n b = self.biases[l]\n\n H1_t = tf.tanh(tf.add(tf.matmul(H1_t, W), b))\n H2_t = tf.tanh(tf.add(tf.matmul(H2_t, W), b))\n H1_x = tf.tanh(tf.add(tf.matmul(H1_x, W), b))\n\n # Merge outputs\n H1 = tf.multiply(H1_t, H1_x)\n H2 = tf.multiply(H2_t, H1_x)\n H = tf.concat([H1, H2], 1)\n\n W = self.weights[-1]\n b = self.biases[-1]\n H = tf.add(tf.matmul(H, W), b)\n\n return H\n\n # Forward pass for u\n def net_u(self, t, x):\n u = self.forward_pass(tf.concat([t, x], 1))\n return u\n\n def net_u_t(self, t, x):\n u_t = tf.gradients(self.net_u(t, x), t)[0] / self.sigma_t\n return u_t\n\n # Forward pass for f\n def net_r(self, t, x):\n u = self.net_u(t, x)\n residual = self.operator(u, t, x,\n self.c,\n self.sigma_t,\n self.sigma_x)\n return residual\n\n # Compute Jacobian for each weights and biases in each layer and retrun a list\n def compute_jacobian(self, f):\n J_list = []\n L = len(self.weights)\n for i in range(L):\n J_w = jacobian(f, self.weights[i])\n J_list.append(J_w)\n\n for i in range(L):\n J_b = jacobian(f, self.biases[i])\n J_list.append(J_b)\n return J_list\n\n # Compute the empirical NTK = J J^T\n def compute_ntk(self, J1_list, D1, J2_list, D2):\n\n N = len(J1_list)\n\n Ker = tf.zeros((D1, D2))\n for k in range(N):\n J1 = tf.reshape(J1_list[k], shape=(D1, -1))\n J2 = tf.reshape(J2_list[k], shape=(D2, -1))\n\n K = tf.matmul(J1, tf.transpose(J2))\n Ker = Ker + K\n return Ker\n\n def fetch_minibatch(self, sampler, N):\n X, Y = sampler.sample(N)\n X = (X - self.mu_X) / self.sigma_X\n return X, Y\n\n # Trains the model by minimizing the MSE loss\n def train(self, nIter=10000, batch_size=128, log_NTK=False, update_weights=False):\n\n start_time = timeit.default_timer()\n for it in range(nIter):\n # Fetch boundary mini-batches\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size // 3)\n X_bc1_batch, _ = self.fetch_minibatch(self.bcs_sampler[0], batch_size // 3)\n X_bc2_batch, _ = self.fetch_minibatch(self.bcs_sampler[1], batch_size // 3)\n\n # Fetch residual mini-batch\n X_res_batch, _ = self.fetch_minibatch(self.res_sampler, batch_size)\n\n # Define a dictionary for associating placeholders with data\n tf_dict = {self.t_ics_tf: X_ics_batch[:, 0:1], self.x_ics_tf: X_ics_batch[:, 1:2],\n self.u_ics_tf: u_ics_batch,\n self.t_bc1_tf: X_bc1_batch[:, 0:1], self.x_bc1_tf: X_bc1_batch[:, 1:2],\n self.t_bc2_tf: X_bc2_batch[:, 0:1], self.x_bc2_tf: X_bc2_batch[:, 1:2],\n self.t_r_tf: X_res_batch[:, 0:1], self.x_r_tf: X_res_batch[:, 1:2],\n self.lambda_u_tf: self.lambda_u_val,\n self.lambda_ut_tf: self.lambda_ut_val,\n self.lambda_r_tf: self.lambda_r_val}\n\n # Run the Tensorflow session to minimize the loss\n self.sess.run(self.train_op, tf_dict)\n\n # Print\n if it % 100 == 0:\n elapsed = timeit.default_timer() - start_time\n\n loss_value = self.sess.run(self.loss, tf_dict)\n loss_bcs_value = self.sess.run(self.loss_bcs, tf_dict)\n loss_ics_ut_value = self.sess.run(self.loss_ics_u_t, tf_dict)\n loss_res_value = self.sess.run(self.loss_res, tf_dict)\n \n u_pred = self.predict_u(self.X_star)\n error = np.linalg.norm(self.u_star - u_pred, 2) / np.linalg.norm(self.u_star, 2)\n\n self.loss_bcs_log.append(loss_bcs_value)\n self.loss_res_log.append(loss_res_value)\n self.loss_ut_ics_log.append(loss_ics_ut_value)\n self.l2_error_log.append(error)\n\n print('It: %d, Loss: %.3e, Loss_res: %.3e, Loss_bcs: %.3e, Loss_ut_ics: %.3e,, Time: %.2f' %\n (it, loss_value, loss_res_value, loss_bcs_value, loss_ics_ut_value, elapsed))\n\n print('lambda_u: {}'.format(self.lambda_u_val))\n print('lambda_ut: {}'.format(self.lambda_ut_val))\n print('lambda_r: {}'.format(self.lambda_r_val))\n\n start_time = timeit.default_timer()\n\n if log_NTK:\n X_bc_batch = np.vstack([X_ics_batch, X_bc1_batch, X_bc2_batch])\n X_ics_batch, u_ics_batch = self.fetch_minibatch(self.ics_sampler, batch_size)\n\n if it % 100 == 0:\n print(\"Compute NTK...\")\n tf_dict = {self.t_u_ntk_tf: X_bc_batch[:, 0:1], self.x_u_ntk_tf: X_bc_batch[:, 1:2],\n self.t_ut_ntk_tf: X_ics_batch[:, 0:1], self.x_ut_ntk_tf: X_ics_batch[:, 1:2],\n self.t_r_ntk_tf: X_res_batch[:, 0:1], self.x_r_ntk_tf: X_res_batch[:, 1:2]}\n\n K_u_value, K_ut_value, K_r_value = self.sess.run([self.K_u, self.K_ut, self.K_r], tf_dict)\n\n\n self.K_u_log.append(K_u_value)\n self.K_ut_log.append(K_ut_value)\n self.K_r_log.append(K_r_value)\n\n if update_weights:\n lambda_K_sum = np.trace(K_u_value) + np.trace(K_ut_value) + \\\n np.trace(K_r_value)\n\n self.lambda_u_val = lambda_K_sum / np.trace(K_u_value)\n self.lambda_ut_val = lambda_K_sum / np.trace(K_ut_value)\n self.lambda_r_val = lambda_K_sum / np.trace(K_r_value)\n\n # Store weights\n self.lambda_u_log.append(self.lambda_u_val)\n self.lambda_ut_log.append(self.lambda_ut_val)\n self.lambda_r_log.append(self.lambda_r_val)\n\n # Evaluates predictions at test points\n def predict_u(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_u_tf: X_star[:, 0:1], self.x_u_tf: X_star[:, 1:2]}\n u_star = self.sess.run(self.u_pred, tf_dict)\n return u_star\n\n # Evaluates predictions at test points\n def predict_r(self, X_star):\n X_star = (X_star - self.mu_X) / self.sigma_X\n tf_dict = {self.t_r_tf: X_star[:, 0:1], self.x_r_tf: X_star[:, 1:2]}\n r_star = self.sess.run(self.r_pred, tf_dict)\n return r_star\n\n\n\n \n\n\n" } ]