[ { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n" }, { "path": "A3C_sparse.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport gym\nimport time\nimport threading\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\n\n# global variables for threading\nstep = 0\nhistory = {'episode': [], 'Episode_reward': []}\nlock = threading.Lock()\n\n\nclass A3C:\n \"\"\"A3C Algorithms with sparse action.\n \"\"\"\n def __init__(self):\n self.gamma = 0.95\n self.actor_lr = 0.001\n self.critic_lr = 0.01\n\n self._build_model()\n self.optimizer = self._build_optimizer()\n\n # handle error\n self.sess = tf.InteractiveSession()\n K.set_session(self.sess)\n self.sess.run(tf.global_variables_initializer())\n\n def _build_actor(self):\n \"\"\"actor model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n x = Dense(1, activation='sigmoid')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def _build_critic(self):\n \"\"\"critic model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n x = Dense(1, activation='linear')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def _build_model(self):\n \"\"\"build model for multi threading training.\n \"\"\"\n self.actor = self._build_actor()\n self.critic = self._build_critic()\n\n # Pre-compile for threading\n self.actor._make_predict_function()\n self.critic._make_predict_function()\n\n def _build_optimizer(self):\n \"\"\"build optimizer and loss method.\n\n Returns:\n [actor optimizer, critic optimizer].\n \"\"\"\n # actor optimizer\n actions = K.placeholder(shape=(None, 1))\n advantages = K.placeholder(shape=(None, 1))\n action_pred = self.actor.output\n\n entropy = K.sum(action_pred * K.log(action_pred + 1e-10), axis=1)\n closs = K.binary_crossentropy(actions, action_pred)\n actor_loss = K.mean(closs * K.flatten(advantages)) - 0.01 * entropy\n\n actor_optimizer = Adam(lr=self.actor_lr)\n actor_updates = actor_optimizer.get_updates(self.actor.trainable_weights, [], actor_loss)\n actor_train = K.function([self.actor.input, actions, advantages], [], updates=actor_updates)\n\n # critic optimizer\n discounted_reward = K.placeholder(shape=(None, 1))\n value = self.critic.output\n\n critic_loss = K.mean(K.square(discounted_reward - value))\n\n critic_optimizer = Adam(lr=self.critic_lr)\n critic_updates = critic_optimizer.get_updates(self.critic.trainable_weights, [], critic_loss)\n critic_train = K.function([self.critic.input, discounted_reward], [], updates=critic_updates)\n\n return [actor_train, critic_train]\n\n def train(self, episode, n_thread, update_iter):\n \"\"\"training A3C.\n\n Arguments:\n episode: total training episode.\n n_thread: number of thread.\n update_iter: update iter.\n \"\"\"\n # Multi threading training.\n threads = [Agent(i, self.actor, self.critic, self.optimizer, self.gamma, episode, update_iter) for i in range(n_thread)]\n\n for t in threads:\n t.start()\n time.sleep(1)\n\n try:\n [t.join() for t in threads]\n except KeyboardInterrupt:\n print(\"Exiting all threads...\")\n\n self.save()\n\n def load(self):\n \"\"\"Load model weights.\n \"\"\"\n if os.path.exists('model/actor_a3cs.h5') and os.path.exists('model/critic_a3cs.h5'):\n self.actor.load_weights('model/actor_a3cs.h5')\n self.critic.load_weights('model/critic_a3cs.h5')\n\n def save(self):\n \"\"\"Save model weights.\n \"\"\"\n self.actor.save_weights('model/actor_a3cs.h5')\n self.critic.save_weights('model/critic_a3cs.h5')\n\n\nclass Agent(threading.Thread):\n \"\"\"Multi threading training agent.\n \"\"\"\n def __init__(self, index, actor, critic, optimizer, gamma, episode, update_iter):\n threading.Thread.__init__(self)\n\n self.index = index\n self.actor = actor\n self.critic = critic\n self.optimizer = optimizer\n self.gamma = gamma\n self.episode = episode\n self.update_iter = update_iter\n\n self.env = gym.make('CartPole-v0')\n\n def run(self):\n \"\"\"training model.\n \"\"\"\n global history\n global step\n\n while step < self.episode:\n observation = self.env.reset()\n\n states = []\n actions = []\n rewards = []\n\n while True:\n x = observation.reshape(-1, 4)\n states.append(x)\n\n # choice action with prob.\n prob = self.actor.predict(x)[0][0]\n action = np.random.choice(np.array(range(2)), p=[1 - prob, prob])\n actions.append(action)\n\n next_observation, reward, done, _ = self.env.step(action)\n next_observation = next_observation.reshape(-1, 4)\n rewards.append(reward)\n\n observation = next_observation[0]\n\n if ((step + 1) % self.update_iter == 0) or done:\n lock.acquire()\n try:\n self.train_episode(states, actions, rewards, next_observation, done)\n\n if done:\n episode_reward = sum(rewards)\n history['episode'].append(step)\n history['Episode_reward'].append(episode_reward)\n\n print('Thread: {} | Episode: {} | Episode reward: {}'.format(self.index, step, episode_reward))\n\n step += 1\n finally:\n lock.release()\n\n if done:\n break\n\n def discount_reward(self, rewards, next_state, done):\n \"\"\"Discount reward\n\n Arguments:\n rewards: rewards in a episode.\n next_states: next state of current game step.\n done: if epsiode done.\n\n Returns:\n discount_reward: n-step discount rewards.\n \"\"\"\n # compute the discounted reward backwards through time.\n discount_rewards = np.zeros_like(rewards, dtype=np.float32)\n\n if done:\n cumulative = 0.\n else: \n cumulative = self.critic.predict(next_state)[0][0]\n\n for i in reversed(range(len(rewards))):\n cumulative = cumulative * self.gamma + rewards[i]\n discount_rewards[i] = cumulative\n\n return discount_rewards\n\n def train_episode(self, states, actions, rewards, next_observation, done):\n \"\"\"training algorithm in an epsiode.\n \"\"\"\n states = np.concatenate(states, axis=0)\n actions = np.array(actions).reshape(-1, 1)\n rewards = np.array(rewards)\n\n # Q_values\n values = self.critic.predict(states)\n # discounted rewards\n discounted_rewards = self.discount_reward(rewards, next_observation, done)\n discounted_rewards = discounted_rewards.reshape(-1, 1)\n # advantages\n advantages = discounted_rewards - values\n\n self.optimizer[1]([states, discounted_rewards])\n self.optimizer[0]([states, actions, advantages])\n\n\ndef save_history(history, name):\n \"\"\"save reward history.\n \"\"\"\n name = os.path.join('history', name)\n\n df = pd.DataFrame.from_dict(history)\n df.to_csv(name, index=False, encoding='utf-8')\n\n\ndef play(model):\n \"\"\"play game with model.\n \"\"\"\n print('play...')\n\n env = gym.make('CartPole-v0')\n observation = env.reset()\n\n reward_sum = 0\n random_episodes = 0\n\n while random_episodes < 10:\n env.render()\n\n prob = model.actor.predict(observation.reshape(-1, 4))[0][0]\n action = 1 if prob > 0.5 else 0\n\n observation, reward, done, _ = env.step(action)\n\n reward_sum += reward\n\n if done:\n print(\"Reward for this episode was: {}\".format(reward_sum))\n random_episodes += 1\n reward_sum = 0\n observation = env.reset()\n\n env.close()\n\n\nif __name__ == '__main__':\n model = A3C()\n#\n# model.train(2000, 4, 10)\n# save_history(history, 'a3c_sparse.csv')\n\n model.load()\n play(model)\n" }, { "path": "AC_continue.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport gym\nimport numpy as np\n\nfrom keras.layers import Input, Dense, concatenate, Lambda\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\nfrom DRL import DRL\n\n\nclass AC(DRL):\n \"\"\"Actor Critic Algorithms with continuous action.\n not stable during training.\n \"\"\"\n def __init__(self):\n super(AC, self).__init__()\n\n self.env = gym.make('Pendulum-v0')\n self.bound = self.env.action_space.high[0]\n\n self.actor = self._build_actor()\n self.critic = self._build_critic()\n\n if os.path.exists('model/actor_acc.h5') and os.path.exists('model/critic_acc.h5'):\n self.actor.load_weights('model/actor_acc.h5')\n self.critic.load_weights('model/critic_acc.h5')\n\n self.gamma = 0.9\n\n def _build_actor(self):\n \"\"\"actor model.\n \"\"\"\n inputs = Input(shape=(3,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n\n mu = Dense(1, activation='tanh')(x)\n mu = Lambda(lambda x: self.bound * x)(mu)\n\n sigma = Dense(1, activation='softplus')(x)\n sigma = Lambda(lambda x: x + 0.0001)(sigma)\n\n out = concatenate([mu, sigma], axis=-1)\n\n model = Model(inputs=inputs, outputs=out)\n\n return model\n\n def _build_critic(self):\n \"\"\"critic model.\n \"\"\"\n inputs = Input(shape=(3,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n x = Dense(1, activation='linear')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def _actor_loss(self, y_true, y_pred):\n \"\"\"actor loss function.\n\n Arguments:\n y_true: (action, reward)\n y_pred: action\n\n Returns:\n loss: reward loss\n \"\"\"\n mu, sigma = y_pred[:, 0], y_pred[:, 1]\n action_true, td_error = y_true[:, 0], y_true[:, 1]\n\n # probability density function\n pdf = 1. / K.sqrt(2. * np.pi * sigma) * K.exp(-K.square(action_true - mu) / (2. * sigma))\n log_pdf = K.log(pdf + K.epsilon())\n # entropy for explore\n entropy = K.sum(0.5 * (K.log(2. * np.pi * sigma) + 1.))\n\n exp_v = log_pdf * td_error\n\n exp_v = K.sum(exp_v + 0.01 * entropy)\n loss = -exp_v\n\n return loss\n\n def discount_reward(self, next_states, reward):\n \"\"\"Discount reward for critic\n\n Arguments:\n next_states: next_states\n rewards: reward of last action.\n done: if game done.\n \"\"\"\n q = self.critic.predict(next_states)[0][0]\n target = reward + self.gamma * q\n\n return target\n\n def choice_action(self, x):\n \"\"\"choice continuous action from normal distributions.\n\n Arguments:\n x: state\n\n Returns:\n action: action\n \"\"\"\n mu, sigma = self.actor.predict(x)[0]\n epsilon = np.random.randn(1)[0]\n action = mu + np.sqrt(sigma) * epsilon\n\n action = np.clip(action, -self.bound, self.bound)\n\n return action\n\n def train(self, episode):\n \"\"\"training model.\n\n Arguments:\n episode: ganme episode\n\n Returns:\n history: training history\n \"\"\"\n self.actor.compile(loss=self._actor_loss, optimizer=Adam(lr=0.001))\n self.critic.compile(loss='mse', optimizer=Adam(lr=0.002))\n\n history = {'episode': [], 'Episode_reward': [],\n 'actor_loss': [], 'critic_loss': []}\n\n for i in range(episode):\n observation = self.env.reset()\n rewards = []\n alosses = []\n closses = []\n\n while True:\n x = observation.reshape(-1, 3)\n\n action = self.choice_action(x)\n next_observation, reward, done, _ = self.env.step([action])\n next_observation = next_observation.reshape(-1, 3)\n\n rewards.append(reward)\n\n target = self.discount_reward(next_observation, reward)\n y = np.array([target])\n # TD_error = (r + gamma * next_q) - current_q\n td_error = target - self.critic.predict(x)[0][0]\n # loss1 = mse((r + gamma * next_q), current_q)\n loss1 = self.critic.train_on_batch(x, y)\n\n y = np.array([[action, td_error]])\n loss2 = self.actor.train_on_batch(x, y)\n\n observation = next_observation[0]\n\n alosses.append(loss2)\n closses.append(loss1)\n\n if done:\n episode_reward = np.sum(rewards)\n aloss = np.mean(alosses)\n closs = np.mean(closses)\n\n history['episode'].append(i)\n history['Episode_reward'].append(episode_reward)\n history['actor_loss'].append(aloss)\n history['critic_loss'].append(closs)\n\n print('Episode: {} | Episode reward: {:.2f} | actor_loss: {:.3f} | critic_loss: {:.3f}'.format(i, episode_reward, aloss, closs))\n\n break\n\n self.actor.save_weights('model/actor_acc.h5')\n self.critic.save_weights('model/critic_acc.h5')\n\n return history\n\n def play(self):\n \"\"\"play game with model.\n \"\"\"\n print('play...')\n observation = self.env.reset()\n\n reward_sum = 0\n random_episodes = 0\n\n while random_episodes < 10:\n self.env.render()\n\n x = observation.reshape(-1, 3)\n action = self.choice_action(x)\n observation, reward, done, _ = self.env.step([action])\n \n reward_sum += reward\n\n if done:\n print(\"Reward for this episode was: {}\".format(reward_sum))\n random_episodes += 1\n reward_sum = 0\n observation = self.env.reset()\n\n self.env.close()\n\n\nif __name__ == '__main__':\n model = AC()\n\n history = model.train(500)\n model.save_history(history, 'ac_continue.csv')\n\n model.play()\n" }, { "path": "AC_sparse.py", "content": "# -*- coding: utf-8 -*-\nimport os\n\nimport numpy as np\n\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\nfrom DRL import DRL\n\n\nclass AC(DRL):\n \"\"\"Actor Critic Algorithms with sparse action.\n \"\"\"\n def __init__(self):\n super(AC, self).__init__()\n\n self.actor = self._build_actor()\n self.critic = self._build_critic()\n\n self.gamma = 0.9\n\n def load(self):\n if os.path.exists('model/actor_acs.h5') and os.path.exists('model/critic_acs.h5'):\n self.actor.load_weights('model/actor_acs.h5')\n self.critic.load_weights('model/critic_acs.h5')\n\n def _build_actor(self):\n \"\"\"actor model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n x = Dense(1, activation='sigmoid')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def _build_critic(self):\n \"\"\"critic model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(20, activation='relu')(inputs)\n x = Dense(20, activation='relu')(x)\n x = Dense(1, activation='linear')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def _actor_loss(self, y_true, y_pred):\n \"\"\"actor loss function.\n\n Arguments:\n y_true: (action, reward)\n y_pred: action_prob\n\n Returns:\n loss: reward loss\n \"\"\"\n action_pred = y_pred\n action_true, td_error = y_true[:, 0], y_true[:, 1]\n action_true = K.reshape(action_true, (-1, 1))\n\n loss = K.binary_crossentropy(action_true, action_pred)\n loss = loss * K.flatten(td_error)\n\n return loss\n\n def discount_reward(self, next_states, reward, done):\n \"\"\"Discount reward for Critic\n\n Arguments:\n next_states: next_states\n rewards: reward of last action.\n done: if game done.\n \"\"\"\n q = self.critic.predict(next_states)[0][0]\n\n target = reward\n if not done:\n target = reward + self.gamma * q\n\n return target\n\n def train(self, episode):\n \"\"\"training model.\n\n Arguments:\n episode: ganme episode\n\n Returns:\n history: training history\n \"\"\"\n self.actor.compile(loss=self._actor_loss, optimizer=Adam(lr=0.001))\n self.critic.compile(loss='mse', optimizer=Adam(lr=0.01))\n\n history = {'episode': [], 'Episode_reward': [],\n 'actor_loss': [], 'critic_loss': []}\n\n for i in range(episode):\n observation = self.env.reset()\n rewards = []\n alosses = []\n closses = []\n\n while True:\n x = observation.reshape(-1, 4)\n # choice action with prob.\n prob = self.actor.predict(x)[0][0]\n action = np.random.choice(np.array(range(2)), p=[1 - prob, prob])\n\n next_observation, reward, done, _ = self.env.step(action)\n next_observation = next_observation.reshape(-1, 4)\n rewards.append(reward)\n\n target = self.discount_reward(next_observation, reward, done)\n y = np.array([target])\n\n # TD_error = (r + gamma * next_q) - current_q\n td_error = target - self.critic.predict(x)[0][0]\n # loss1 = mse((r + gamma * next_q), current_q)\n loss1 = self.critic.train_on_batch(x, y)\n\n y = np.array([[action, td_error]])\n loss2 = self.actor.train_on_batch(x, y)\n\n observation = next_observation[0]\n\n alosses.append(loss2)\n closses.append(loss1)\n\n if done:\n episode_reward = sum(rewards)\n aloss = np.mean(alosses)\n closs = np.mean(closses)\n\n history['episode'].append(i)\n history['Episode_reward'].append(episode_reward)\n history['actor_loss'].append(aloss)\n history['critic_loss'].append(closs)\n\n print('Episode: {} | Episode reward: {} | actor_loss: {:.3f} | critic_loss: {:.3f}'.format(i, episode_reward, aloss, closs))\n\n break\n\n self.actor.save_weights('model/actor_acs.h5')\n self.critic.save_weights('model/critic_acs.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = AC()\n\n history = model.train(300)\n model.save_history(history, 'ac_sparse.csv')\n \n model.load()\n model.play('acs')\n" }, { "path": "DDPG.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport random\nimport gym\nfrom collections import deque\n\nimport numpy as np\nimport tensorflow as tf\n\nfrom keras.layers import Input, Dense, Lambda, concatenate\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\nfrom DRL import DRL\n\n\nclass DDPG(DRL):\n \"\"\"Deep Deterministic Policy Gradient Algorithms.\n \"\"\"\n def __init__(self):\n super(DDPG, self).__init__()\n\n self.sess = K.get_session()\n self.env = gym.make('Pendulum-v0')\n self.bound = self.env.action_space.high[0]\n\n # update rate for target model.\n self.TAU = 0.01\n # experience replay.\n self.memory_buffer = deque(maxlen=4000)\n # discount rate for q value.\n self.gamma = 0.95\n # epsilon of action selection\n self.epsilon = 1.0\n # discount rate for epsilon.\n self.epsilon_decay = 0.995\n # min epsilon of ε-greedy.\n self.epsilon_min = 0.01\n\n # actor learning rate\n self.a_lr = 0.0001\n # critic learining rate\n self.c_lr = 0.001\n\n # ddpg model\n self.actor = self._build_actor()\n self.critic = self._build_critic()\n\n # target model\n self.target_actor = self._build_actor()\n self.target_actor.set_weights(self.actor.get_weights())\n self.target_critic = self._build_critic()\n self.target_critic.set_weights(self.critic.get_weights())\n\n # gradient function\n\n self.get_critic_grad = self.critic_gradient()\n self.actor_optimizer()\n\n def load(self):\n if os.path.exists('model/ddpg_actor.h5') and os.path.exists('model/ddpg_critic.h5'):\n self.actor.load_weights('model/ddpg_actor.h5')\n self.critic.load_weights('model/ddpg_critic.h5')\n\n def _build_actor(self):\n \"\"\"Actor model.\n \"\"\"\n inputs = Input(shape=(3,), name='state_input')\n x = Dense(40, activation='relu')(inputs)\n x = Dense(40, activation='relu')(x)\n x = Dense(1, activation='tanh')(x)\n output = Lambda(lambda x: x * self.bound)(x)\n\n model = Model(inputs=inputs, outputs=output)\n model.compile(loss='mse', optimizer=Adam(lr=self.a_lr))\n\n return model\n\n def _build_critic(self):\n \"\"\"Critic model.\n \"\"\"\n sinput = Input(shape=(3,), name='state_input')\n ainput = Input(shape=(1,), name='action_input')\n s = Dense(40, activation='relu')(sinput)\n a = Dense(40, activation='relu')(ainput)\n x = concatenate([s, a])\n x = Dense(40, activation='relu')(x)\n output = Dense(1, activation='linear')(x)\n\n model = Model(inputs=[sinput, ainput], outputs=output)\n model.compile(loss='mse', optimizer=Adam(lr=self.c_lr))\n\n return model\n\n def actor_optimizer(self):\n \"\"\"actor_optimizer.\n\n Returns:\n function, opt function for actor.\n \"\"\"\n self.ainput = self.actor.input\n aoutput = self.actor.output\n trainable_weights = self.actor.trainable_weights\n self.action_gradient = tf.placeholder(tf.float32, shape=(None, 1))\n\n # tf.gradients will calculate dy/dx with a initial gradients for y\n # action_gradient is dq / da, so this is dq/da * da/dparams\n params_grad = tf.gradients(aoutput, trainable_weights, -self.action_gradient)\n grads = zip(params_grad, trainable_weights)\n self.opt = tf.train.AdamOptimizer(self.a_lr).apply_gradients(grads)\n self.sess.run(tf.global_variables_initializer())\n\n def critic_gradient(self):\n \"\"\"get critic gradient function.\n\n Returns:\n function, gradient function for critic.\n \"\"\"\n cinput = self.critic.input\n coutput = self.critic.output\n\n # compute the gradient of the action with q value, dq/da.\n action_grads = K.gradients(coutput, cinput[1])\n\n return K.function([cinput[0], cinput[1]], action_grads)\n\n def OU(self, x, mu=0, theta=0.15, sigma=0.2):\n \"\"\"Ornstein-Uhlenbeck process.\n formula：ou = θ * (μ - x) + σ * w\n\n Arguments:\n x: action value.\n mu: μ, mean fo values.\n theta: θ, rate the variable reverts towards to the mean. \n sigma：σ, degree of volatility of the process.\n\n Returns:\n OU value\n \"\"\"\n return theta * (mu - x) + sigma * np.random.randn(1)\n\n def get_action(self, X):\n \"\"\"get actor action with ou noise.\n\n Arguments:\n X: state value.\n \"\"\"\n action = self.actor.predict(X)[0][0]\n\n # add randomness to action selection for exploration\n noise = max(self.epsilon, 0) * self.OU(action)\n action = np.clip(action + noise, -self.bound, self.bound)\n\n return action\n\n def remember(self, state, action, reward, next_state, done):\n \"\"\"add data to experience replay.\n\n Arguments:\n state: observation.\n action: action.\n reward: reward.\n next_state: next_observation.\n done: if game done.\n \"\"\"\n item = (state, action, reward, next_state, done)\n self.memory_buffer.append(item)\n\n def update_epsilon(self):\n \"\"\"update epsilon.\n \"\"\"\n if self.epsilon >= self.epsilon_min:\n self.epsilon *= self.epsilon_decay\n\n def process_batch(self, batch):\n \"\"\"process batch data.\n\n Arguments:\n batch: batch size.\n\n Returns:\n states: states.\n actions: actions.\n y: Q_value.\n \"\"\"\n y = []\n # ranchom choice batch data from experience replay.\n data = random.sample(self.memory_buffer, batch)\n states = np.array([d[0] for d in data])\n actions = np.array([d[1] for d in data])\n next_states = np.array([d[3] for d in data])\n\n # Q_target。\n next_actions = self.target_actor.predict(next_states)\n q = self.target_critic.predict([next_states, next_actions])\n\n # update Q value\n for i, (_, _, reward, _, done) in enumerate(data):\n target = reward\n if not done:\n target += self.gamma * q[i][0]\n y.append(target)\n\n return states, actions, y\n\n def update_model(self, X1, X2, y):\n \"\"\"update ddpg model.\n\n Arguments:\n states: states.\n actions: actions.\n y: Q_value.\n\n Returns:\n loss: critic loss.\n \"\"\"\n# loss = self.critic.train_on_batch([X1, X2], y)\n loss = self.critic.fit([X1, X2], y, verbose=0)\n loss = np.mean(loss.history['loss'])\n\n X3 = self.actor.predict(X1)\n a_grads = np.array(self.get_critic_grad([X1, X3]))[0]\n self.sess.run(self.opt, feed_dict={\n self.ainput: X1,\n self.action_gradient: a_grads\n })\n\n return loss\n\n def update_target_model(self):\n \"\"\"soft update target model.\n formula：θ​​t ← τ * θ + (1−τ) * θt, τ << 1. \n \"\"\"\n critic_weights = self.critic.get_weights()\n actor_weights = self.actor.get_weights()\n critic_target_weights = self.target_critic.get_weights()\n actor_target_weights = self.target_actor.get_weights()\n\n for i in range(len(critic_weights)):\n critic_target_weights[i] = self.TAU * critic_weights[i] + (1 - self.TAU) * critic_target_weights[i]\n\n for i in range(len(actor_weights)):\n actor_target_weights[i] = self.TAU * actor_weights[i] + (1 - self.TAU) * actor_target_weights[i]\n\n self.target_critic.set_weights(critic_target_weights)\n self.target_actor.set_weights(actor_target_weights)\n\n def train(self, episode, batch):\n \"\"\"training model.\n Arguments:\n episode: ganme episode.\n batch： batch size of episode.\n\n Returns:\n history: training history.\n \"\"\"\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n for i in range(episode):\n observation = self.env.reset()\n reward_sum = 0\n losses = []\n\n for j in range(200):\n # chocie action from ε-greedy.\n x = observation.reshape(-1, 3)\n\n # actor action\n action = self.get_action(x)\n observation, reward, done, _ = self.env.step(action)\n # add data to experience replay.\n reward_sum += reward\n self.remember(x[0], action, reward, observation, done)\n\n if len(self.memory_buffer) > batch:\n X1, X2, y = self.process_batch(batch)\n\n # update DDPG model\n loss = self.update_model(X1, X2, y)\n # update target model\n self.update_target_model()\n # reduce epsilon pure batch.\n self.update_epsilon()\n\n losses.append(loss)\n\n loss = np.mean(losses)\n history['episode'].append(i)\n history['Episode_reward'].append(reward_sum)\n history['Loss'].append(loss)\n\n print('Episode: {}/{} | reward: {} | loss: {:.3f}'.format(i, episode, reward_sum, loss))\n\n self.actor.save_weights('model/ddpg_actor.h5')\n self.critic.save_weights('model/ddpg_critic.h5')\n\n return history\n\n def play(self):\n \"\"\"play game with model.\n \"\"\"\n print('play...')\n observation = self.env.reset()\n\n reward_sum = 0\n random_episodes = 0\n\n while random_episodes < 10:\n self.env.render()\n\n x = observation.reshape(-1, 3)\n action = self.actor.predict(x)[0]\n observation, reward, done, _ = self.env.step(action)\n\n reward_sum += reward\n\n if done:\n print(\"Reward for this episode was: {}\".format(reward_sum))\n random_episodes += 1\n reward_sum = 0\n observation = self.env.reset()\n\n self.env.close()\n\n\nif __name__ == '__main__':\n model = DDPG()\n\n history = model.train(200, 128)\n model.save_history(history, 'ddpg.csv')\n \n model.load()\n model.play()\n" }, { "path": "DQN.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport random\nimport numpy as np\n\nfrom collections import deque\n\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.optimizers import Adam\n\nfrom DRL import DRL\n\n\nclass DQN(DRL):\n \"\"\"Deep Q-Learning.\n \"\"\"\n def __init__(self):\n super(DQN, self).__init__()\n\n self.model = self.build_model()\n\n # experience replay.\n self.memory_buffer = deque(maxlen=2000)\n # discount rate for q value.\n self.gamma = 0.95\n # epsilon of ε-greedy.\n self.epsilon = 1.0\n # discount rate for epsilon.\n self.epsilon_decay = 0.995\n # min epsilon of ε-greedy.\n self.epsilon_min = 0.01\n\n def load(self):\n if os.path.exists('model/dqn.h5'):\n self.model.load_weights('model/dqn.h5')\n\n def build_model(self):\n \"\"\"basic model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(16, activation='relu')(inputs)\n x = Dense(16, activation='relu')(x)\n x = Dense(2, activation='linear')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n model.compile(loss='mse', optimizer=Adam(1e-3))\n\n return model\n\n def egreedy_action(self, state):\n \"\"\"ε-greedy\n Arguments:\n state: observation\n\n Returns:\n action: action\n \"\"\"\n if np.random.rand() <= self.epsilon:\n return random.randint(0, 1)\n else:\n q_values = self.model.predict(state)[0]\n return np.argmax(q_values)\n\n def remember(self, state, action, reward, next_state, done):\n \"\"\"add data to experience replay.\n Arguments:\n state: observation\n action: action\n reward: reward\n next_state: next_observation\n done: if game done.\n \"\"\"\n item = (state, action, reward, next_state, done)\n self.memory_buffer.append(item)\n\n def update_epsilon(self):\n \"\"\"update epsilon\n \"\"\"\n if self.epsilon >= self.epsilon_min:\n self.epsilon *= self.epsilon_decay\n\n def process_batch(self, batch):\n \"\"\"process batch data\n Arguments:\n batch: batch size\n\n Returns:\n X: states\n y: [Q_value1, Q_value2]\n \"\"\"\n # ranchom choice batch data from experience replay.\n data = random.sample(self.memory_buffer, batch)\n # Q_target。\n states = np.array([d[0] for d in data])\n next_states = np.array([d[3] for d in data])\n\n y = self.model.predict(states)\n q = self.model.predict(next_states)\n\n for i, (_, action, reward, _, done) in enumerate(data):\n target = reward\n if not done:\n target += self.gamma * np.amax(q[i])\n y[i][action] = target\n\n return states, y\n\n\n def train(self, episode, batch):\n \"\"\"training \n Arguments:\n episode: game episode\n batch： batch size\n\n Returns:\n history: training history\n \"\"\"\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n count = 0\n for i in range(episode):\n observation = self.env.reset()\n reward_sum = 0\n loss = np.infty\n done = False\n\n while not done:\n # chocie action from ε-greedy.\n x = observation.reshape(-1, 4)\n action = self.egreedy_action(x)\n observation, reward, done, _ = self.env.step(action)\n # add data to experience replay.\n reward_sum += reward\n self.remember(x[0], action, reward, observation, done)\n\n if len(self.memory_buffer) > batch:\n X, y = self.process_batch(batch)\n loss = self.model.train_on_batch(X, y)\n\n count += 1\n # reduce epsilon pure batch.\n self.update_epsilon()\n\n if i % 5 == 0:\n history['episode'].append(i)\n history['Episode_reward'].append(reward_sum)\n history['Loss'].append(loss)\n\n print('Episode: {} | Episode reward: {} | loss: {:.3f} | e:{:.2f}'.format(i, reward_sum, loss, self.epsilon))\n\n self.model.save_weights('model/dqn.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = DQN()\n\n history = model.train(600, 32)\n model.save_history(history, 'dqn.csv')\n \n model.load()\n model.play()\n" }, { "path": "DRL.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport gym\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n\nclass DRL:\n def __init__(self):\n self.env = gym.make('CartPole-v0')\n\n if not os.path.exists('model'):\n os.mkdir('model')\n\n if not os.path.exists('history'):\n os.mkdir('history')\n\n def play(self, m='pg'):\n \"\"\"play game with model.\n \"\"\"\n print('play...')\n observation = self.env.reset()\n\n reward_sum = 0\n random_episodes = 0\n\n while random_episodes < 10:\n self.env.render()\n\n x = observation.reshape(-1, 4)\n if m == 'pg':\n prob = self.model.predict(x)[0][0]\n action = 1 if prob > 0.5 else 0\n elif m == 'acs':\n prob = self.actor.predict(x)[0][0]\n action = 1 if prob > 0.5 else 0\n else:\n action = np.argmax(self.model.predict(x)[0])\n observation, reward, done, _ = self.env.step(action)\n\n reward_sum += reward\n\n if done:\n print(\"Reward for this episode was: {}\".format(reward_sum))\n random_episodes += 1\n reward_sum = 0\n observation = self.env.reset()\n\n self.env.close()\n\n def plot(self, history):\n x = history['episode']\n r = history['Episode_reward']\n l = history['Loss']\n\n fig = plt.figure()\n ax = fig.add_subplot(121)\n ax.plot(x, r)\n ax.set_title('Episode_reward')\n ax.set_xlabel('episode')\n ax = fig.add_subplot(122)\n ax.plot(x, l)\n ax.set_title('Loss')\n ax.set_xlabel('episode')\n\n plt.show()\n\n def save_history(self, history, name):\n name = os.path.join('history', name)\n\n df = pd.DataFrame.from_dict(history)\n df.to_csv(name, index=False, encoding='utf-8')\n" }, { "path": "DoubleDQN.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport random\nimport numpy as np\n\nfrom DQN import DQN\n\n\nclass DDQN(DQN):\n \"\"\"Nature Deep Q-Learning.\n \"\"\"\n def __init__(self):\n super(DDQN, self).__init__()\n\n self.model = self.build_model()\n self.target_model = self.build_model()\n self.update_target_model()\n\n def load(self):\n if os.path.exists('model/ddqn.h5'):\n self.model.load_weights('model/ddqn.h5')\n\n def update_target_model(self):\n \"\"\"update target_model\n \"\"\"\n self.target_model.set_weights(self.model.get_weights())\n\n def process_batch(self, batch):\n \"\"\"process batch data\n Arguments:\n batch: batch size\n\n Returns:\n X: states\n y: [Q_value1, Q_value2]\n \"\"\"\n # ranchom choice batch data from experience replay.\n data = random.sample(self.memory_buffer, batch)\n # Q_target。\n states = np.array([d[0] for d in data])\n next_states = np.array([d[3] for d in data])\n\n y = self.model.predict(states)\n q = self.target_model.predict(next_states)\n next_action = np.argmax(self.model.predict(next_states), axis=1)\n\n for i, (_, action, reward, _, done) in enumerate(data):\n target = reward\n if not done:\n target += self.gamma * q[i][next_action[i]]\n y[i][action] = target\n\n return states, y\n\n def train(self, episode, batch):\n \"\"\"training \n Arguments:\n episode: game episode\n batch： batch size\n\n Returns:\n history: training history\n \"\"\"\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n count = 0\n for i in range(episode):\n observation = self.env.reset()\n reward_sum = 0\n loss = np.infty\n done = False\n\n while not done:\n # chocie action from ε-greedy.\n x = observation.reshape(-1, 4)\n action = self.egreedy_action(x)\n observation, reward, done, _ = self.env.step(action)\n # add data to experience replay.\n reward_sum += reward\n self.remember(x[0], action, reward, observation, done)\n\n if len(self.memory_buffer) > batch:\n X, y = self.process_batch(batch)\n loss = self.model.train_on_batch(X, y)\n\n count += 1\n # reduce epsilon pure batch.\n self.update_epsilon()\n\n # update target_model every 20 episode\n if count != 0 and count % 20 == 0:\n self.update_target_model()\n\n if i % 5 == 0:\n history['episode'].append(i)\n history['Episode_reward'].append(reward_sum)\n history['Loss'].append(loss)\n\n print('Episode: {} | Episode reward: {} | loss: {:.3f} | e:{:.2f}'.format(i, reward_sum, loss, self.epsilon))\n\n self.model.save_weights('model/ddqn.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = DDQN()\n\n history = model.train(600, 32)\n model.save_history(history, 'ddqn.csv')\n\n model.load()\n model.play('dqn')\n" }, { "path": "DuelingDQN.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport numpy as np\n\nfrom keras.layers import Input, Dense, Add, Subtract, Lambda\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\nfrom NatureDQN import NDQN\n\n\nclass DuelingDQN(NDQN):\n \"\"\"Dueling DQN.\n \"\"\"\n def __init__(self):\n super(DuelingDQN, self).__init__()\n\n def load(self):\n if os.path.exists('model/dueling.h5'):\n self.model.load_weights('model/dueling.h5')\n\n def build_model(self):\n \"\"\"basic model.\n \"\"\"\n inputs = Input(shape=(4,))\n x = Dense(16, activation='relu')(inputs)\n x = Dense(16, activation='relu')(x)\n\n value = Dense(2, activation='linear')(x)\n a = Dense(2, activation='linear')(x)\n meam = Lambda(lambda x: K.mean(x, axis=1, keepdims=True))(a)\n advantage = Subtract()([a, meam])\n\n q = Add()([value, advantage])\n\n model = Model(inputs=inputs, outputs=q)\n\n model.compile(loss='mse', optimizer=Adam(1e-3))\n\n return model\n\n def train(self, episode, batch):\n \"\"\"training \n Arguments:\n episode: game episode\n batch： batch size\n\n Returns:\n history: training history\n \"\"\"\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n count = 0\n for i in range(episode):\n observation = self.env.reset()\n reward_sum = 0\n loss = np.infty\n done = False\n\n while not done:\n # chocie action from ε-greedy.\n x = observation.reshape(-1, 4)\n action = self.egreedy_action(x)\n observation, reward, done, _ = self.env.step(action)\n # add data to experience replay.\n reward_sum += reward\n self.remember(x[0], action, reward, observation, done)\n\n if len(self.memory_buffer) > batch:\n X, y = self.process_batch(batch)\n loss = self.model.train_on_batch(X, y)\n\n count += 1\n # reduce epsilon pure batch.\n self.update_epsilon()\n\n # update target_model every 20 episode\n if count != 0 and count % 20 == 0:\n self.update_target_model()\n\n if i % 5 == 0:\n history['episode'].append(i)\n history['Episode_reward'].append(reward_sum)\n history['Loss'].append(loss)\n\n print('Episode: {} | Episode reward: {} | loss: {:.3f} | e:{:.2f}'.format(i, reward_sum, loss, self.epsilon))\n\n self.model.save_weights('model/dueling.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = DuelingDQN()\n\n history = model.train(600, 32)\n model.save_history(history, 'dueling.csv')\n\n model.load()\n model.play('dqn')\n" }, { "path": "NatureDQN.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport random\nimport numpy as np\n\nfrom DQN import DQN\n\n\nclass NDQN(DQN):\n \"\"\"Nature Deep Q-Learning.\n \"\"\"\n def __init__(self):\n super(NDQN, self).__init__()\n\n self.model = self.build_model()\n self.target_model = self.build_model()\n self.update_target_model()\n\n def load(self):\n if os.path.exists('model/ndqn.h5'):\n self.model.load_weights('model/ndqn.h5')\n\n def update_target_model(self):\n \"\"\"update target_model\n \"\"\"\n self.target_model.set_weights(self.model.get_weights())\n\n def process_batch(self, batch):\n \"\"\"process batch data\n Arguments:\n batch: batch size\n\n Returns:\n X: states\n y: [Q_value1, Q_value2]\n \"\"\"\n # ranchom choice batch data from experience replay.\n data = random.sample(self.memory_buffer, batch)\n # Q_target。\n states = np.array([d[0] for d in data])\n next_states = np.array([d[3] for d in data])\n\n y = self.model.predict(states)\n q = self.target_model.predict(next_states)\n\n for i, (_, action, reward, _, done) in enumerate(data):\n target = reward\n if not done:\n target += self.gamma * np.amax(q[i])\n y[i][action] = target\n\n return states, y\n\n def train(self, episode, batch):\n \"\"\"training \n Arguments:\n episode: game episode\n batch： batch size\n\n Returns:\n history: training history\n \"\"\"\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n count = 0\n for i in range(episode):\n observation = self.env.reset()\n reward_sum = 0\n loss = np.infty\n done = False\n\n while not done:\n # chocie action from ε-greedy.\n x = observation.reshape(-1, 4)\n action = self.egreedy_action(x)\n observation, reward, done, _ = self.env.step(action)\n # add data to experience replay.\n reward_sum += reward\n self.remember(x[0], action, reward, observation, done)\n\n if len(self.memory_buffer) > batch:\n X, y = self.process_batch(batch)\n loss = self.model.train_on_batch(X, y)\n\n count += 1\n # reduce epsilon pure batch.\n self.update_epsilon()\n\n # update target_model every 20 episode\n if count != 0 and count % 20 == 0:\n self.update_target_model()\n\n if i % 5 == 0:\n history['episode'].append(i)\n history['Episode_reward'].append(reward_sum)\n history['Loss'].append(loss)\n\n print('Episode: {} | Episode reward: {} | loss: {:.3f} | e:{:.2f}'.format(i, reward_sum, loss, self.epsilon))\n\n self.model.save_weights('model/ndqn.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = NDQN()\n\n history = model.train(600, 32)\n model.save_history(history, 'ndqn.csv')\n\n model.load()\n model.play('dqn')\n" }, { "path": "PPO_TF.py", "content": "import os\nimport gym\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\n\nclass PPO:\n def __init__(self, ep, batch, t='ppo2'):\n self.t = t\n self.ep = ep\n self.batch = batch\n self.log = 'model/{}_log'.format(t)\n\n self.env = gym.make('Pendulum-v0')\n self.bound = self.env.action_space.high[0]\n\n self.gamma = 0.9\n self.A_LR = 0.0001\n self.C_LR = 0.0002\n self.A_UPDATE_STEPS = 10\n self.C_UPDATE_STEPS = 10\n\n # KL penalty, d_target、β for ppo1\n self.kl_target = 0.01\n self.lam = 0.5\n # ε for ppo2\n self.epsilon = 0.2\n\n self.sess = tf.Session()\n self.build_model()\n\n def _build_critic(self):\n \"\"\"critic model.\n \"\"\"\n with tf.variable_scope('critic'):\n x = tf.layers.dense(self.states, 100, tf.nn.relu)\n\n self.v = tf.layers.dense(x, 1)\n self.advantage = self.dr - self.v\n\n def _build_actor(self, name, trainable):\n \"\"\"actor model.\n \"\"\"\n with tf.variable_scope(name):\n x = tf.layers.dense(self.states, 100, tf.nn.relu, trainable=trainable)\n\n mu = self.bound * tf.layers.dense(x, 1, tf.nn.tanh, trainable=trainable)\n sigma = tf.layers.dense(x, 1, tf.nn.softplus, trainable=trainable)\n\n norm_dist = tf.distributions.Normal(loc=mu, scale=sigma)\n\n params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)\n\n return norm_dist, params\n\n def build_model(self):\n \"\"\"build model with ppo loss.\n \"\"\"\n # inputs\n self.states = tf.placeholder(tf.float32, [None, 3], 'states')\n self.action = tf.placeholder(tf.float32, [None, 1], 'action')\n self.adv = tf.placeholder(tf.float32, [None, 1], 'advantage')\n self.dr = tf.placeholder(tf.float32, [None, 1], 'discounted_r')\n\n # build model\n self._build_critic()\n nd, pi_params = self._build_actor('actor', trainable=True)\n old_nd, oldpi_params = self._build_actor('old_actor', trainable=False)\n\n # define ppo loss\n with tf.variable_scope('loss'):\n # critic loss\n self.closs = tf.reduce_mean(tf.square(self.advantage))\n\n # actor loss\n with tf.variable_scope('surrogate'):\n ratio = tf.exp(nd.log_prob(self.action) - old_nd.log_prob(self.action))\n surr = ratio * self.adv\n\n if self.t == 'ppo1':\n self.tflam = tf.placeholder(tf.float32, None, 'lambda')\n kl = tf.distributions.kl_divergence(old_nd, nd)\n self.kl_mean = tf.reduce_mean(kl)\n self.aloss = -(tf.reduce_mean(surr - self.tflam * kl))\n else: \n self.aloss = -tf.reduce_mean(tf.minimum(\n surr,\n tf.clip_by_value(ratio, 1.- self.epsilon, 1.+ self.epsilon) * self.adv))\n\n # define Optimizer\n with tf.variable_scope('optimize'):\n self.ctrain_op = tf.train.AdamOptimizer(self.C_LR).minimize(self.closs)\n self.atrain_op = tf.train.AdamOptimizer(self.A_LR).minimize(self.aloss)\n\n with tf.variable_scope('sample_action'):\n self.sample_op = tf.squeeze(nd.sample(1), axis=0)\n\n # update old actor\n with tf.variable_scope('update_old_actor'):\n self.update_old_actor = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)]\n\n tf.summary.FileWriter(self.log, self.sess.graph)\n\n self.sess.run(tf.global_variables_initializer())\n\n def choose_action(self, state):\n \"\"\"choice continuous action from normal distributions.\n\n Arguments:\n state: state.\n\n Returns:\n action.\n \"\"\"\n state = state[np.newaxis, :]\n action = self.sess.run(self.sample_op, {self.states: state})[0]\n\n return np.clip(action, -self.bound, self.bound)\n\n def get_value(self, state):\n \"\"\"get q value.\n\n Arguments:\n state: state.\n\n Returns:\n q_value.\n \"\"\"\n if state.ndim < 2: state = state[np.newaxis, :]\n\n return self.sess.run(self.v, {self.states: state})\n\n def discount_reward(self, states, rewards, next_observation):\n \"\"\"Compute target value.\n\n Arguments:\n states: state in episode.\n rewards: reward in episode.\n next_observation: state of last action.\n\n Returns:\n targets: q targets.\n \"\"\"\n s = np.vstack([states, next_observation.reshape(-1, 3)])\n q_values = self.get_value(s).flatten()\n\n targets = rewards + self.gamma * q_values[1:]\n targets = targets.reshape(-1, 1)\n\n return targets\n\n# not work.\n# def neglogp(self, mean, std, x):\n# \"\"\"Gaussian likelihood\n# \"\"\"\n# return 0.5 * tf.reduce_sum(tf.square((x - mean) / std), axis=-1) \\\n# + 0.5 * np.log(2.0 * np.pi) * tf.to_float(tf.shape(x)[-1]) \\\n# + tf.reduce_sum(tf.log(std), axis=-1)\n\n def update(self, states, action, dr):\n \"\"\"update model.\n\n Arguments:\n states: states.\n action: action of states.\n dr: discount reward of action.\n \"\"\"\n self.sess.run(self.update_old_actor)\n\n adv = self.sess.run(self.advantage,\n {self.states: states,\n self.dr: dr})\n\n # update actor\n if self.t == 'ppo1':\n # run ppo1 loss\n for _ in range(self.A_UPDATE_STEPS):\n _, kl = self.sess.run(\n [self.atrain_op, self.kl_mean],\n {self.states: states,\n self.action: action,\n self.adv: adv,\n self.tflam: self.lam})\n\n if kl < self.kl_target / 1.5:\n self.lam /= 2\n elif kl > self.kl_target * 1.5:\n self.lam *= 2\n else:\n # run ppo2 loss\n for _ in range(self.A_UPDATE_STEPS):\n self.sess.run(self.atrain_op,\n {self.states: states,\n self.action: action,\n self.adv: adv})\n\n # update critic\n for _ in range(self.C_UPDATE_STEPS):\n self.sess.run(self.ctrain_op,\n {self.states: states,\n self.dr: dr})\n\n def train(self):\n \"\"\"train method.\n \"\"\"\n tf.reset_default_graph()\n\n history = {'episode': [], 'Episode_reward': []}\n\n for i in range(self.ep):\n observation = self.env.reset()\n\n states, actions, rewards = [], [], []\n episode_reward = 0\n j = 0\n\n while True:\n a = self.choose_action(observation)\n next_observation, reward, done, _ = self.env.step(a)\n states.append(observation)\n actions.append(a)\n\n episode_reward += reward\n rewards.append((reward + 8) / 8)\n\n observation = next_observation\n\n if (j + 1) % self.batch == 0:\n states = np.array(states)\n actions = np.array(actions)\n rewards = np.array(rewards)\n d_reward = self.discount_reward(states, rewards, next_observation)\n\n self.update(states, actions, d_reward)\n\n states, actions, rewards = [], [], []\n\n if done:\n break\n j += 1\n\n history['episode'].append(i)\n history['Episode_reward'].append(episode_reward)\n print('Episode: {} | Episode reward: {:.2f}'.format(i, episode_reward))\n\n return history\n\n def save_history(self, history, name):\n name = os.path.join('history', name)\n\n df = pd.DataFrame.from_dict(history)\n df.to_csv(name, index=False, encoding='utf-8')\n\n\nif __name__ == '__main__':\n model = PPO(1000, 32, 'ppo2')\n history = model.train()\n model.save_history(history, 'ppo2.csv')\n" }, { "path": "PolicyNetwork.py", "content": "# -*- coding: utf-8 -*-\nimport os\n\nimport numpy as np\n\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nimport keras.backend as K\n\nfrom DRL import DRL\n\n\nclass PolicyNetwork(DRL):\n \"\"\"Policy Gradient Algorithms(Policy Network)\n \"\"\"\n def __init__(self):\n super(PolicyNetwork, self).__init__()\n\n self.model = self._build_model()\n self.gamma = 0.95\n\n def load(self):\n if os.path.exists('model/pg.h5'):\n self.model.load_weights('model/pg.h5')\n\n def _build_model(self):\n \"\"\"basic model.\n \"\"\"\n inputs = Input(shape=(4,), name='ob_input')\n x = Dense(16, activation='relu')(inputs)\n x = Dense(16, activation='relu')(x)\n x = Dense(1, activation='sigmoid')(x)\n\n model = Model(inputs=inputs, outputs=x)\n\n return model\n\n def loss(self, y_true, y_pred):\n \"\"\"loss function.\n Arguments:\n y_true: (action, reward)\n y_pred: action_prob\n\n Returns:\n loss: reward loss\n \"\"\"\n action_pred = y_pred\n action_true, discount_episode_reward = y_true[:, 0], y_true[:, 1]\n\n action_true = K.reshape(action_true, (-1, 1))\n loss = K.binary_crossentropy(action_true, action_pred)\n loss = loss * K.flatten(discount_episode_reward)\n\n return loss\n\n def discount_reward(self, rewards):\n \"\"\"Discount reward\n Arguments:\n rewards: rewards in a episode.\n \"\"\"\n # compute the discounted reward backwards through time.\n discount_rewards = np.zeros_like(rewards, dtype=np.float32)\n cumulative = 0.\n for i in reversed(range(len(rewards))):\n cumulative = cumulative * self.gamma + rewards[i]\n discount_rewards[i] = cumulative\n\n # size the rewards to be unit normal (helps control the gradient estimator variance).\n discount_rewards -= np.mean(discount_rewards)\n discount_rewards //= np.std(discount_rewards)\n\n return list(discount_rewards)\n\n def train(self, episode, batch):\n \"\"\"training model.\n Arguments:\n episode: ganme episode\n batch： batch size of episode\n\n Returns:\n history: training history\n \"\"\"\n self.model.compile(loss=self.loss, optimizer=Adam(lr=0.01))\n\n history = {'episode': [], 'Episode_reward': [], 'Loss': []}\n\n episode_reward = 0\n states = []\n actions = []\n rewards = []\n discount_rewards = []\n\n for i in range(episode):\n observation = self.env.reset()\n erewards = []\n\n while True:\n x = observation.reshape(-1, 4)\n prob = self.model.predict(x)[0][0]\n\n # choice action with prob.\n action = np.random.choice(np.array(range(2)), size=1, p=[1 - prob, prob])[0]\n observation, reward, done, _ = self.env.step(action)\n\n states.append(x[0])\n actions.append(action)\n erewards.append(reward)\n rewards.append(reward)\n\n if done:\n # calculate discount rewards every episode.\n discount_rewards.extend(self.discount_reward(erewards))\n break\n\n if i != 0 and i % batch == 0: \n episode_reward = sum(rewards) / batch\n\n X = np.array(states)\n y = np.array(list(zip(actions, discount_rewards)))\n\n loss = self.model.train_on_batch(X, y)\n\n history['episode'].append(i)\n history['Episode_reward'].append(episode_reward)\n history['Loss'].append(loss)\n\n print('Episode: {} | Episode reward: {} | loss: {:.3f}'.format(i, episode_reward, loss))\n\n episode_reward = 0\n states = []\n actions = []\n rewards = []\n discount_rewards = []\n\n self.model.save_weights('model/pg.h5')\n\n return history\n\n\nif __name__ == '__main__':\n model = PolicyNetwork()\n\n history = model.train(5000, 5)\n model.save_history(history, 'pg.csv')\n \n model.load()\n model.play()\n" }, { "path": "README.md", "content": "# Deep-Reinforcement-Learning-Practice\nPractice of Deep Reinforcement Learning with Keras and gym.\n\nContinuous updating...\n\n\n## Algorithm\n\n\n| #\t| Name | Paper |\n| - | ----- | :--------: |\n| 1 | [DQN](DQN.py) | [Playing Atari with Deep Reinforcement Learning](http://cn.arxiv.org/abs/1312.5602)|\n| 2 | [Nature DQN](NatureDQN.py) | [Human-level control through deep reinforcement learning](https://www.nature.com/articles/nature14236) |\n| 3 | [Double DQN](DoubleDQN.py) | [Deep Reinforcement Learning with Double Q-learning](http://cn.arxiv.org/abs/1509.06461v1) |\n| 4 | [Dueling DQN](DuelingDQN.py) | [Dueling Network Architectures for Deep Reinforcement Learning](https://arxiv.org/pdf/1511.06581.pdf) |\n| 5 | [Actor-Critic](AC_sparse.py) | [Actor-Critic Algorithms](https://papers.nips.cc/paper/1786-actor-critic-algorithms.pdf) |\n| 6 | [Policy Network](PolicyNetwork.py) | [Policy gradient methods for reinforcement learning with function approximation](https://www.researchgate.net/publication/2503757_Policy_Gradient_Methods_for_Reinforcement_Learning_with_Function_Approximation) |\n| 7 | [DDPG](DDPG.py) | [Continuous Control with Deep Reinforcement Learning](https://arxiv.org/abs/1509.02971) |\n| 8 | [PPO_TF](PPO_TF.py) | [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347) |\n| 9 | [A3C](A3C_sparse.py) | [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/pdf/1602.01783.pdf) |\n" }, { "path": "game/CartPole.py", "content": "# -*- coding: utf-8 -*-\nimport gym\nimport numpy as np\n\n\ndef try_gym():\n # creat CartPole env.\n env = gym.make('CartPole-v0')\n # reset game env.\n env.reset()\n\n # episodes of game\n random_episodes = 0\n # sum of reward of game per episode\n reward_sum = 0\n\n while random_episodes < 10:\n # show game\n env.render()\n # random choice a action\n # execute the action\n observation, reward, done, _ = env.step(np.random.randint(0, 2))\n reward_sum += reward\n # print result and reset ganme env if game done.\n if done:\n random_episodes += 1\n print(\"Reward for this episode was: {}\".format(reward_sum))\n reward_sum = 0\n env.reset()\n\n env.close()\n\n\nif __name__ == '__main__':\n try_gym()\n" }, { "path": "game/Pendulum.py", "content": "# -*- coding: utf-8 -*-\nimport gym\n\n\ndef try_gym():\n # creat Pendulum env.\n env = gym.make('Pendulum-v0')\n # reset game env.\n env.reset()\n\n # episodes of game\n random_episodes = 0\n # sum of reward of game per episode\n reward_sum = 0\n\n while random_episodes < 10:\n # show game\n env.render()\n # random choice a action\n # execute the action\n action = env.action_space.sample()\n observation, reward, done, _ = env.step(action)\n reward_sum += reward\n # print result and reset ganme env if game done.\n if done:\n random_episodes += 1\n print(\"Reward for this episode was: {}\".format(reward_sum))\n reward_sum = 0\n env.reset()\n\n env.close()\n\n\nif __name__ == '__main__':\n try_gym()\n" }, { "path": "history/a3c_sparse.csv", "content": "episode,Episode_reward\n0,11.0\n1,17.0\n2,17.0\n3,13.0\n4,14.0\n5,22.0\n6,34.0\n7,14.0\n8,20.0\n9,25.0\n10,14.0\n11,12.0\n12,14.0\n13,15.0\n14,11.0\n15,21.0\n16,33.0\n17,18.0\n18,20.0\n19,34.0\n20,13.0\n21,25.0\n22,21.0\n23,17.0\n24,19.0\n25,26.0\n26,36.0\n27,45.0\n28,31.0\n29,19.0\n30,14.0\n31,23.0\n32,32.0\n33,19.0\n34,10.0\n35,14.0\n36,15.0\n37,28.0\n38,10.0\n39,56.0\n40,20.0\n41,10.0\n42,82.0\n43,32.0\n44,10.0\n45,21.0\n46,67.0\n47,27.0\n48,21.0\n49,37.0\n50,15.0\n51,19.0\n52,27.0\n53,45.0\n54,18.0\n55,22.0\n56,25.0\n57,17.0\n58,31.0\n59,26.0\n60,33.0\n61,40.0\n62,14.0\n63,41.0\n64,76.0\n65,23.0\n66,83.0\n67,32.0\n68,77.0\n69,69.0\n70,21.0\n71,17.0\n72,51.0\n73,17.0\n74,18.0\n75,111.0\n76,11.0\n77,38.0\n78,31.0\n79,31.0\n80,37.0\n81,25.0\n82,51.0\n83,17.0\n84,28.0\n85,18.0\n86,21.0\n87,84.0\n88,54.0\n89,42.0\n90,24.0\n91,28.0\n92,65.0\n93,60.0\n94,20.0\n95,52.0\n96,22.0\n97,124.0\n98,35.0\n99,90.0\n100,113.0\n101,66.0\n102,108.0\n103,118.0\n104,103.0\n105,61.0\n106,172.0\n107,121.0\n108,39.0\n109,63.0\n110,35.0\n111,38.0\n112,64.0\n113,141.0\n114,47.0\n115,154.0\n116,75.0\n117,31.0\n118,22.0\n119,24.0\n120,60.0\n121,42.0\n122,65.0\n123,70.0\n124,28.0\n125,38.0\n126,87.0\n127,30.0\n128,39.0\n129,60.0\n130,145.0\n131,31.0\n132,115.0\n133,34.0\n134,23.0\n135,19.0\n136,35.0\n137,29.0\n138,139.0\n139,51.0\n140,70.0\n141,55.0\n142,55.0\n143,78.0\n144,70.0\n145,156.0\n146,87.0\n147,57.0\n148,111.0\n149,46.0\n150,98.0\n151,65.0\n152,30.0\n153,128.0\n154,128.0\n155,21.0\n156,79.0\n157,23.0\n158,117.0\n159,66.0\n160,128.0\n161,144.0\n162,81.0\n163,93.0\n164,137.0\n165,200.0\n166,188.0\n167,146.0\n168,178.0\n169,132.0\n170,76.0\n171,182.0\n172,57.0\n173,200.0\n174,115.0\n175,165.0\n176,148.0\n177,48.0\n178,92.0\n179,38.0\n180,62.0\n181,78.0\n182,63.0\n183,50.0\n184,200.0\n185,89.0\n186,92.0\n187,141.0\n188,103.0\n189,99.0\n190,168.0\n191,118.0\n192,111.0\n193,150.0\n194,126.0\n195,102.0\n196,81.0\n197,92.0\n198,101.0\n199,157.0\n200,142.0\n201,145.0\n202,121.0\n203,200.0\n204,101.0\n205,38.0\n206,93.0\n207,55.0\n208,66.0\n209,115.0\n210,111.0\n211,33.0\n212,141.0\n213,180.0\n214,183.0\n215,200.0\n216,200.0\n217,199.0\n218,200.0\n219,109.0\n220,200.0\n221,200.0\n222,200.0\n223,200.0\n224,150.0\n225,200.0\n226,143.0\n227,163.0\n228,126.0\n229,151.0\n230,191.0\n231,146.0\n232,160.0\n233,159.0\n234,181.0\n235,133.0\n236,159.0\n237,196.0\n238,171.0\n239,170.0\n240,200.0\n241,200.0\n242,200.0\n243,197.0\n244,200.0\n245,147.0\n246,200.0\n247,200.0\n248,200.0\n249,167.0\n250,200.0\n251,200.0\n252,200.0\n253,164.0\n254,159.0\n255,200.0\n256,115.0\n257,200.0\n258,192.0\n259,154.0\n260,138.0\n261,58.0\n262,161.0\n263,200.0\n264,200.0\n265,150.0\n266,145.0\n267,188.0\n268,154.0\n269,86.0\n270,151.0\n271,53.0\n272,117.0\n273,106.0\n274,70.0\n275,45.0\n276,65.0\n277,105.0\n278,42.0\n279,38.0\n280,69.0\n281,59.0\n282,86.0\n283,52.0\n284,75.0\n285,23.0\n286,99.0\n287,66.0\n288,43.0\n289,70.0\n290,55.0\n291,105.0\n292,85.0\n293,112.0\n294,84.0\n295,76.0\n296,83.0\n297,146.0\n298,114.0\n299,87.0\n300,107.0\n301,118.0\n302,168.0\n303,163.0\n304,200.0\n305,200.0\n306,186.0\n307,200.0\n308,200.0\n309,182.0\n310,133.0\n311,200.0\n312,121.0\n313,34.0\n314,96.0\n315,20.0\n316,180.0\n317,30.0\n318,20.0\n319,21.0\n320,17.0\n321,22.0\n322,27.0\n323,26.0\n324,89.0\n325,102.0\n326,27.0\n327,31.0\n328,41.0\n329,107.0\n330,20.0\n331,111.0\n332,127.0\n333,94.0\n334,134.0\n335,103.0\n336,187.0\n337,200.0\n338,200.0\n339,200.0\n340,200.0\n341,149.0\n342,200.0\n343,200.0\n344,112.0\n345,146.0\n346,200.0\n347,153.0\n348,200.0\n349,123.0\n350,200.0\n351,152.0\n352,168.0\n353,200.0\n354,131.0\n355,147.0\n356,123.0\n357,148.0\n358,200.0\n359,200.0\n360,175.0\n361,200.0\n362,130.0\n363,200.0\n364,200.0\n365,120.0\n366,33.0\n367,173.0\n368,147.0\n369,175.0\n370,200.0\n371,200.0\n372,200.0\n373,188.0\n374,110.0\n375,200.0\n376,116.0\n377,104.0\n378,200.0\n379,109.0\n380,200.0\n381,191.0\n382,200.0\n383,200.0\n384,200.0\n385,200.0\n386,128.0\n387,170.0\n388,200.0\n389,200.0\n390,200.0\n391,200.0\n392,127.0\n393,200.0\n394,192.0\n395,200.0\n396,171.0\n397,200.0\n398,123.0\n399,200.0\n400,191.0\n401,173.0\n402,148.0\n403,198.0\n404,96.0\n405,200.0\n406,110.0\n407,199.0\n408,200.0\n409,106.0\n410,189.0\n411,200.0\n412,200.0\n413,181.0\n414,200.0\n415,200.0\n416,200.0\n417,197.0\n418,189.0\n419,124.0\n420,162.0\n421,197.0\n422,200.0\n423,200.0\n424,200.0\n425,200.0\n426,200.0\n427,200.0\n428,200.0\n429,188.0\n430,83.0\n431,197.0\n432,200.0\n433,200.0\n434,200.0\n435,200.0\n436,165.0\n437,200.0\n438,200.0\n439,200.0\n440,200.0\n441,200.0\n442,200.0\n443,143.0\n444,200.0\n445,200.0\n446,192.0\n447,200.0\n448,200.0\n449,200.0\n450,164.0\n451,135.0\n452,160.0\n453,119.0\n454,140.0\n455,139.0\n456,154.0\n457,167.0\n458,148.0\n459,139.0\n460,122.0\n461,129.0\n462,131.0\n463,117.0\n464,145.0\n465,159.0\n466,127.0\n467,157.0\n468,156.0\n469,166.0\n470,148.0\n471,141.0\n472,200.0\n473,200.0\n474,183.0\n475,200.0\n476,200.0\n477,200.0\n478,200.0\n479,200.0\n480,200.0\n481,200.0\n482,200.0\n483,200.0\n484,200.0\n485,200.0\n486,200.0\n487,200.0\n488,200.0\n489,200.0\n490,200.0\n491,200.0\n492,200.0\n493,200.0\n494,200.0\n495,200.0\n496,183.0\n497,185.0\n498,200.0\n499,200.0\n500,200.0\n501,200.0\n502,173.0\n503,200.0\n504,166.0\n505,200.0\n506,194.0\n507,200.0\n508,200.0\n509,200.0\n510,158.0\n511,200.0\n512,187.0\n513,166.0\n514,184.0\n515,142.0\n516,179.0\n517,139.0\n518,184.0\n519,181.0\n520,136.0\n521,194.0\n522,191.0\n523,156.0\n524,131.0\n525,177.0\n526,128.0\n527,163.0\n528,147.0\n529,200.0\n530,200.0\n531,200.0\n532,200.0\n533,200.0\n534,200.0\n535,200.0\n536,200.0\n537,200.0\n538,200.0\n539,200.0\n540,200.0\n541,200.0\n542,200.0\n543,187.0\n544,200.0\n545,200.0\n546,200.0\n547,200.0\n548,200.0\n549,200.0\n550,199.0\n551,200.0\n552,200.0\n553,200.0\n554,200.0\n555,200.0\n556,200.0\n557,200.0\n558,200.0\n559,142.0\n560,200.0\n561,200.0\n562,143.0\n563,200.0\n564,200.0\n565,181.0\n566,166.0\n567,200.0\n568,160.0\n569,200.0\n570,200.0\n571,200.0\n572,200.0\n573,200.0\n574,200.0\n575,200.0\n576,200.0\n577,200.0\n578,200.0\n579,200.0\n580,200.0\n581,200.0\n582,200.0\n583,200.0\n584,200.0\n585,200.0\n586,200.0\n587,200.0\n588,200.0\n589,200.0\n590,200.0\n591,200.0\n592,200.0\n593,200.0\n594,200.0\n595,200.0\n596,200.0\n597,200.0\n598,200.0\n599,200.0\n600,200.0\n601,200.0\n602,200.0\n603,200.0\n604,200.0\n605,200.0\n606,200.0\n607,200.0\n608,200.0\n609,200.0\n610,200.0\n611,200.0\n612,200.0\n613,200.0\n614,200.0\n615,200.0\n616,200.0\n617,200.0\n618,200.0\n619,200.0\n620,200.0\n621,200.0\n622,200.0\n623,200.0\n624,200.0\n625,200.0\n626,200.0\n627,200.0\n628,200.0\n629,200.0\n630,200.0\n631,200.0\n632,200.0\n633,200.0\n634,200.0\n635,200.0\n636,200.0\n637,200.0\n638,200.0\n639,200.0\n640,200.0\n641,200.0\n642,200.0\n643,200.0\n644,200.0\n645,200.0\n646,200.0\n647,200.0\n648,200.0\n649,200.0\n650,200.0\n651,200.0\n652,200.0\n653,200.0\n654,200.0\n655,200.0\n656,200.0\n657,200.0\n658,200.0\n659,200.0\n660,200.0\n661,200.0\n662,200.0\n663,200.0\n664,200.0\n665,200.0\n666,200.0\n667,200.0\n668,200.0\n669,200.0\n670,200.0\n671,200.0\n672,200.0\n673,200.0\n674,200.0\n675,200.0\n676,200.0\n677,200.0\n678,200.0\n679,200.0\n680,200.0\n681,200.0\n682,200.0\n683,182.0\n684,175.0\n685,200.0\n686,200.0\n687,200.0\n688,159.0\n689,160.0\n690,169.0\n691,186.0\n692,183.0\n693,162.0\n694,170.0\n695,153.0\n696,195.0\n697,170.0\n698,170.0\n699,200.0\n700,174.0\n701,167.0\n702,137.0\n703,184.0\n704,147.0\n705,168.0\n706,162.0\n707,138.0\n708,130.0\n709,143.0\n710,129.0\n711,135.0\n712,133.0\n713,130.0\n714,137.0\n715,115.0\n716,127.0\n717,136.0\n718,130.0\n719,122.0\n720,140.0\n721,87.0\n722,47.0\n723,51.0\n724,30.0\n725,41.0\n726,42.0\n727,32.0\n728,34.0\n729,29.0\n730,24.0\n731,26.0\n732,25.0\n733,20.0\n734,29.0\n735,26.0\n736,30.0\n737,29.0\n738,29.0\n739,30.0\n740,27.0\n741,26.0\n742,31.0\n743,17.0\n744,26.0\n745,25.0\n746,20.0\n747,25.0\n748,21.0\n749,31.0\n750,19.0\n751,21.0\n752,34.0\n753,15.0\n754,25.0\n755,21.0\n756,22.0\n757,13.0\n758,22.0\n759,31.0\n760,29.0\n761,26.0\n762,26.0\n763,27.0\n764,33.0\n765,31.0\n766,29.0\n767,32.0\n768,34.0\n769,30.0\n770,30.0\n771,38.0\n772,39.0\n773,52.0\n774,34.0\n775,34.0\n776,47.0\n777,36.0\n778,34.0\n779,43.0\n780,34.0\n781,44.0\n782,45.0\n783,42.0\n784,31.0\n785,29.0\n786,42.0\n787,39.0\n788,32.0\n789,37.0\n790,34.0\n791,41.0\n792,52.0\n793,47.0\n794,35.0\n795,42.0\n796,51.0\n797,40.0\n798,44.0\n799,37.0\n800,35.0\n801,40.0\n802,65.0\n803,45.0\n804,49.0\n805,37.0\n806,55.0\n807,67.0\n808,46.0\n809,70.0\n810,49.0\n811,63.0\n812,57.0\n813,50.0\n814,70.0\n815,54.0\n816,71.0\n817,49.0\n818,69.0\n819,85.0\n820,68.0\n821,68.0\n822,80.0\n823,70.0\n824,58.0\n825,66.0\n826,69.0\n827,70.0\n828,55.0\n829,74.0\n830,96.0\n831,74.0\n832,88.0\n833,87.0\n834,87.0\n835,77.0\n836,97.0\n837,82.0\n838,86.0\n839,86.0\n840,82.0\n841,79.0\n842,79.0\n843,92.0\n844,96.0\n845,96.0\n846,88.0\n847,117.0\n848,89.0\n849,133.0\n850,109.0\n851,90.0\n852,108.0\n853,116.0\n854,119.0\n855,101.0\n856,100.0\n857,86.0\n858,121.0\n859,94.0\n860,133.0\n861,94.0\n862,83.0\n863,106.0\n864,101.0\n865,119.0\n866,112.0\n867,102.0\n868,110.0\n869,108.0\n870,124.0\n871,120.0\n872,127.0\n873,103.0\n874,84.0\n875,101.0\n876,92.0\n877,150.0\n878,112.0\n879,114.0\n880,116.0\n881,114.0\n882,107.0\n883,134.0\n884,121.0\n885,98.0\n886,120.0\n887,138.0\n888,106.0\n889,191.0\n890,158.0\n891,200.0\n892,162.0\n893,200.0\n894,194.0\n895,169.0\n896,191.0\n897,169.0\n898,159.0\n899,181.0\n900,147.0\n901,196.0\n902,198.0\n903,155.0\n904,198.0\n905,188.0\n906,190.0\n907,170.0\n908,189.0\n909,200.0\n910,191.0\n911,163.0\n912,156.0\n913,152.0\n914,156.0\n915,140.0\n916,174.0\n917,127.0\n918,140.0\n919,119.0\n920,150.0\n921,156.0\n922,124.0\n923,125.0\n924,137.0\n925,125.0\n926,128.0\n927,153.0\n928,121.0\n929,128.0\n930,119.0\n931,136.0\n932,126.0\n933,143.0\n934,140.0\n935,153.0\n936,158.0\n937,152.0\n938,154.0\n939,128.0\n940,152.0\n941,148.0\n942,177.0\n943,200.0\n944,157.0\n945,182.0\n946,166.0\n947,162.0\n948,171.0\n949,200.0\n950,176.0\n951,194.0\n952,169.0\n953,200.0\n954,192.0\n955,189.0\n956,200.0\n957,200.0\n958,186.0\n959,191.0\n960,190.0\n961,200.0\n962,200.0\n963,200.0\n964,200.0\n965,200.0\n966,200.0\n967,200.0\n968,200.0\n969,200.0\n970,200.0\n971,200.0\n972,200.0\n973,200.0\n974,200.0\n975,200.0\n976,200.0\n977,200.0\n978,200.0\n979,200.0\n980,200.0\n981,200.0\n982,200.0\n983,197.0\n984,200.0\n985,200.0\n986,200.0\n987,200.0\n988,200.0\n989,200.0\n990,200.0\n991,200.0\n992,200.0\n993,200.0\n994,200.0\n995,200.0\n996,200.0\n997,200.0\n998,200.0\n999,200.0\n1000,196.0\n1001,200.0\n1002,200.0\n1003,200.0\n1004,200.0\n1005,200.0\n1006,200.0\n1007,200.0\n1008,200.0\n1009,200.0\n1010,200.0\n1011,200.0\n1012,200.0\n1013,200.0\n1014,200.0\n1015,200.0\n1016,200.0\n1017,200.0\n1018,200.0\n1019,200.0\n1020,200.0\n1021,200.0\n1022,200.0\n1023,200.0\n1024,200.0\n1025,200.0\n1026,200.0\n1027,200.0\n1028,200.0\n1029,200.0\n1030,200.0\n1031,200.0\n1032,200.0\n1033,200.0\n1034,200.0\n1035,200.0\n1036,200.0\n1037,200.0\n1038,200.0\n1039,200.0\n1040,200.0\n1041,200.0\n1042,200.0\n1043,200.0\n1044,200.0\n1045,200.0\n1046,200.0\n1047,200.0\n1048,200.0\n1049,200.0\n1050,200.0\n1051,200.0\n1052,200.0\n1053,200.0\n1054,200.0\n1055,200.0\n1056,200.0\n1057,200.0\n1058,200.0\n1059,200.0\n1060,200.0\n1061,200.0\n1062,200.0\n1063,200.0\n1064,200.0\n1065,200.0\n1066,200.0\n1067,200.0\n1068,200.0\n1069,200.0\n1070,200.0\n1071,200.0\n1072,200.0\n1073,200.0\n1074,200.0\n1075,200.0\n1076,200.0\n1077,200.0\n1078,200.0\n1079,200.0\n1080,200.0\n1081,200.0\n1082,200.0\n1083,200.0\n1084,200.0\n1085,200.0\n1086,200.0\n1087,200.0\n1088,200.0\n1089,200.0\n1090,200.0\n1091,200.0\n1092,200.0\n1093,200.0\n1094,200.0\n1095,200.0\n1096,200.0\n1097,200.0\n1098,200.0\n1099,200.0\n1100,200.0\n1101,200.0\n1102,200.0\n1103,200.0\n1104,200.0\n1105,200.0\n1106,200.0\n1107,200.0\n1108,200.0\n1109,200.0\n1110,200.0\n1111,200.0\n1112,200.0\n1113,200.0\n1114,200.0\n1115,200.0\n1116,200.0\n1117,200.0\n1118,200.0\n1119,200.0\n1120,200.0\n1121,200.0\n1122,200.0\n1123,200.0\n1124,200.0\n1125,200.0\n1126,200.0\n1127,200.0\n1128,200.0\n1129,200.0\n1130,200.0\n1131,200.0\n1132,200.0\n1133,200.0\n1134,200.0\n1135,200.0\n1136,200.0\n1137,200.0\n1138,200.0\n1139,200.0\n1140,200.0\n1141,200.0\n1142,200.0\n1143,200.0\n1144,200.0\n1145,200.0\n1146,200.0\n1147,200.0\n1148,200.0\n1149,200.0\n1150,200.0\n1151,200.0\n1152,200.0\n1153,200.0\n1154,200.0\n1155,200.0\n1156,200.0\n1157,200.0\n1158,200.0\n1159,200.0\n1160,200.0\n1161,200.0\n1162,200.0\n1163,200.0\n1164,200.0\n1165,200.0\n1166,200.0\n1167,200.0\n1168,200.0\n1169,200.0\n1170,200.0\n1171,200.0\n1172,200.0\n1173,200.0\n1174,200.0\n1175,200.0\n1176,200.0\n1177,200.0\n1178,200.0\n1179,200.0\n1180,200.0\n1181,200.0\n1182,200.0\n1183,200.0\n1184,200.0\n1185,200.0\n1186,200.0\n1187,200.0\n1188,200.0\n1189,200.0\n1190,200.0\n1191,200.0\n1192,200.0\n1193,200.0\n1194,200.0\n1195,200.0\n1196,200.0\n1197,200.0\n1198,200.0\n1199,200.0\n1200,200.0\n1201,200.0\n1202,200.0\n1203,200.0\n1204,200.0\n1205,200.0\n1206,200.0\n1207,200.0\n1208,200.0\n1209,200.0\n1210,200.0\n1211,200.0\n1212,200.0\n1213,200.0\n1214,200.0\n1215,200.0\n1216,200.0\n1217,200.0\n1218,200.0\n1219,200.0\n1220,200.0\n1221,200.0\n1222,200.0\n1223,200.0\n1224,200.0\n1225,200.0\n1226,197.0\n1227,200.0\n1228,191.0\n1229,200.0\n1230,200.0\n1231,198.0\n1232,200.0\n1233,200.0\n1234,200.0\n1235,200.0\n1236,200.0\n1237,200.0\n1238,199.0\n1239,200.0\n1240,191.0\n1241,188.0\n1242,179.0\n1243,197.0\n1244,188.0\n1245,183.0\n1246,194.0\n1247,169.0\n1248,166.0\n1249,167.0\n1250,192.0\n1251,170.0\n1252,186.0\n1253,191.0\n1254,181.0\n1255,178.0\n1256,179.0\n1257,189.0\n1258,179.0\n1259,193.0\n1260,177.0\n1261,166.0\n1262,191.0\n1263,200.0\n1264,182.0\n1265,192.0\n1266,198.0\n1267,193.0\n1268,183.0\n1269,190.0\n1270,200.0\n1271,181.0\n1272,200.0\n1273,200.0\n1274,200.0\n1275,200.0\n1276,200.0\n1277,200.0\n1278,200.0\n1279,200.0\n1280,200.0\n1281,200.0\n1282,200.0\n1283,200.0\n1284,200.0\n1285,200.0\n1286,200.0\n1287,200.0\n1288,200.0\n1289,200.0\n1290,199.0\n1291,200.0\n1292,200.0\n1293,200.0\n1294,200.0\n1295,200.0\n1296,200.0\n1297,200.0\n1298,200.0\n1299,200.0\n1300,200.0\n1301,200.0\n1302,185.0\n1303,194.0\n1304,188.0\n1305,183.0\n1306,190.0\n1307,200.0\n1308,186.0\n1309,188.0\n1310,189.0\n1311,190.0\n1312,198.0\n1313,200.0\n1314,199.0\n1315,196.0\n1316,200.0\n1317,191.0\n1318,200.0\n1319,200.0\n1320,200.0\n1321,200.0\n1322,200.0\n1323,200.0\n1324,200.0\n1325,200.0\n1326,200.0\n1327,200.0\n1328,200.0\n1329,200.0\n1330,200.0\n1331,200.0\n1332,200.0\n1333,200.0\n1334,200.0\n1335,200.0\n1336,200.0\n1337,200.0\n1338,200.0\n1339,200.0\n1340,200.0\n1341,200.0\n1342,200.0\n1343,200.0\n1344,200.0\n1345,200.0\n1346,200.0\n1347,200.0\n1348,200.0\n1349,200.0\n1350,200.0\n1351,200.0\n1352,200.0\n1353,200.0\n1354,200.0\n1355,200.0\n1356,200.0\n1357,200.0\n1358,200.0\n1359,200.0\n1360,200.0\n1361,200.0\n1362,200.0\n1363,200.0\n1364,200.0\n1365,200.0\n1366,200.0\n1367,200.0\n1368,200.0\n1369,200.0\n1370,200.0\n1371,200.0\n1372,196.0\n1373,200.0\n1374,200.0\n1375,200.0\n1376,200.0\n1377,200.0\n1378,200.0\n1379,200.0\n1380,188.0\n1381,173.0\n1382,150.0\n1383,143.0\n1384,138.0\n1385,149.0\n1386,146.0\n1387,128.0\n1388,144.0\n1389,152.0\n1390,151.0\n1391,138.0\n1392,157.0\n1393,165.0\n1394,161.0\n1395,168.0\n1396,165.0\n1397,155.0\n1398,165.0\n1399,176.0\n1400,174.0\n1401,182.0\n1402,155.0\n1403,173.0\n1404,174.0\n1405,170.0\n1406,170.0\n1407,176.0\n1408,176.0\n1409,177.0\n1410,180.0\n1411,167.0\n1412,177.0\n1413,177.0\n1414,178.0\n1415,184.0\n1416,179.0\n1417,171.0\n1418,176.0\n1419,182.0\n1420,183.0\n1421,180.0\n1422,170.0\n1423,186.0\n1424,179.0\n1425,182.0\n1426,182.0\n1427,181.0\n1428,193.0\n1429,192.0\n1430,182.0\n1431,191.0\n1432,179.0\n1433,177.0\n1434,180.0\n1435,175.0\n1436,168.0\n1437,187.0\n1438,181.0\n1439,194.0\n1440,188.0\n1441,184.0\n1442,188.0\n1443,195.0\n1444,176.0\n1445,190.0\n1446,189.0\n1447,172.0\n1448,186.0\n1449,181.0\n1450,200.0\n1451,190.0\n1452,191.0\n1453,200.0\n1454,186.0\n1455,175.0\n1456,187.0\n1457,200.0\n1458,186.0\n1459,198.0\n1460,182.0\n1461,181.0\n1462,200.0\n1463,198.0\n1464,181.0\n1465,200.0\n1466,200.0\n1467,200.0\n1468,200.0\n1469,200.0\n1470,199.0\n1471,195.0\n1472,200.0\n1473,200.0\n1474,200.0\n1475,200.0\n1476,200.0\n1477,199.0\n1478,196.0\n1479,200.0\n1480,200.0\n1481,200.0\n1482,188.0\n1483,200.0\n1484,200.0\n1485,186.0\n1486,188.0\n1487,196.0\n1488,188.0\n1489,185.0\n1490,183.0\n1491,189.0\n1492,200.0\n1493,199.0\n1494,200.0\n1495,196.0\n1496,200.0\n1497,200.0\n1498,198.0\n1499,200.0\n1500,198.0\n1501,200.0\n1502,200.0\n1503,200.0\n1504,200.0\n1505,200.0\n1506,200.0\n1507,200.0\n1508,200.0\n1509,179.0\n1510,185.0\n1511,171.0\n1512,187.0\n1513,179.0\n1514,170.0\n1515,187.0\n1516,184.0\n1517,179.0\n1518,179.0\n1519,192.0\n1520,185.0\n1521,189.0\n1522,188.0\n1523,200.0\n1524,194.0\n1525,200.0\n1526,200.0\n1527,196.0\n1528,199.0\n1529,200.0\n1530,200.0\n1531,200.0\n1532,200.0\n1533,200.0\n1534,200.0\n1535,200.0\n1536,200.0\n1537,200.0\n1538,200.0\n1539,200.0\n1540,198.0\n1541,200.0\n1542,200.0\n1543,200.0\n1544,200.0\n1545,200.0\n1546,200.0\n1547,200.0\n1548,200.0\n1549,200.0\n1550,200.0\n1551,200.0\n1552,200.0\n1553,200.0\n1554,200.0\n1555,200.0\n1556,200.0\n1557,200.0\n1558,200.0\n1559,200.0\n1560,200.0\n1561,200.0\n1562,200.0\n1563,200.0\n1564,200.0\n1565,200.0\n1566,200.0\n1567,200.0\n1568,200.0\n1569,200.0\n1570,200.0\n1571,200.0\n1572,200.0\n1573,200.0\n1574,200.0\n1575,200.0\n1576,200.0\n1577,200.0\n1578,200.0\n1579,200.0\n1580,200.0\n1581,200.0\n1582,200.0\n1583,200.0\n1584,200.0\n1585,200.0\n1586,200.0\n1587,200.0\n1588,200.0\n1589,200.0\n1590,200.0\n1591,200.0\n1592,200.0\n1593,200.0\n1594,200.0\n1595,200.0\n1596,200.0\n1597,200.0\n1598,200.0\n1599,200.0\n1600,200.0\n1601,200.0\n1602,200.0\n1603,200.0\n1604,200.0\n1605,200.0\n1606,200.0\n1607,200.0\n1608,200.0\n1609,200.0\n1610,200.0\n1611,200.0\n1612,200.0\n1613,200.0\n1614,200.0\n1615,200.0\n1616,200.0\n1617,200.0\n1618,200.0\n1619,200.0\n1620,200.0\n1621,200.0\n1622,200.0\n1623,200.0\n1624,200.0\n1625,200.0\n1626,200.0\n1627,200.0\n1628,200.0\n1629,200.0\n1630,200.0\n1631,200.0\n1632,200.0\n1633,200.0\n1634,200.0\n1635,200.0\n1636,200.0\n1637,200.0\n1638,200.0\n1639,200.0\n1640,200.0\n1641,200.0\n1642,200.0\n1643,200.0\n1644,200.0\n1645,200.0\n1646,200.0\n1647,200.0\n1648,200.0\n1649,200.0\n1650,200.0\n1651,200.0\n1652,200.0\n1653,200.0\n1654,200.0\n1655,200.0\n1656,200.0\n1657,200.0\n1658,200.0\n1659,200.0\n1660,200.0\n1661,200.0\n1662,200.0\n1663,200.0\n1664,200.0\n1665,200.0\n1666,200.0\n1667,200.0\n1668,200.0\n1669,200.0\n1670,200.0\n1671,200.0\n1672,200.0\n1673,200.0\n1674,200.0\n1675,200.0\n1676,200.0\n1677,200.0\n1678,200.0\n1679,200.0\n1680,200.0\n1681,200.0\n1682,200.0\n1683,200.0\n1684,200.0\n1685,200.0\n1686,200.0\n1687,200.0\n1688,200.0\n1689,200.0\n1690,200.0\n1691,200.0\n1692,200.0\n1693,200.0\n1694,200.0\n1695,200.0\n1696,200.0\n1697,200.0\n1698,200.0\n1699,200.0\n1700,200.0\n1701,200.0\n1702,200.0\n1703,200.0\n1704,200.0\n1705,200.0\n1706,200.0\n1707,200.0\n1708,200.0\n1709,200.0\n1710,200.0\n1711,200.0\n1712,200.0\n1713,200.0\n1714,200.0\n1715,200.0\n1716,200.0\n1717,200.0\n1718,200.0\n1719,200.0\n1720,200.0\n1721,200.0\n1722,200.0\n1723,200.0\n1724,200.0\n1725,200.0\n1726,200.0\n1727,200.0\n1728,200.0\n1729,200.0\n1730,200.0\n1731,200.0\n1732,200.0\n1733,200.0\n1734,200.0\n1735,200.0\n1736,200.0\n1737,200.0\n1738,200.0\n1739,200.0\n1740,200.0\n1741,200.0\n1742,200.0\n1743,200.0\n1744,200.0\n1745,200.0\n1746,200.0\n1747,200.0\n1748,200.0\n1749,200.0\n1750,200.0\n1751,200.0\n1752,200.0\n1753,200.0\n1754,200.0\n1755,200.0\n1756,200.0\n1757,200.0\n1758,200.0\n1759,200.0\n1760,200.0\n1761,200.0\n1762,200.0\n1763,200.0\n1764,200.0\n1765,200.0\n1766,200.0\n1767,200.0\n1768,200.0\n1769,200.0\n1770,200.0\n1771,200.0\n1772,200.0\n1773,200.0\n1774,200.0\n1775,200.0\n1776,200.0\n1777,200.0\n1778,200.0\n1779,200.0\n1780,200.0\n1781,200.0\n1782,200.0\n1783,200.0\n1784,200.0\n1785,200.0\n1786,200.0\n1787,200.0\n1788,200.0\n1789,200.0\n1790,200.0\n1791,200.0\n1792,200.0\n1793,200.0\n1794,200.0\n1795,200.0\n1796,200.0\n1797,200.0\n1798,200.0\n1799,200.0\n1800,200.0\n1801,200.0\n1802,200.0\n1803,200.0\n1804,200.0\n1805,200.0\n1806,200.0\n1807,200.0\n1808,200.0\n1809,200.0\n1810,200.0\n1811,200.0\n1812,200.0\n1813,200.0\n1814,200.0\n1815,200.0\n1816,200.0\n1817,200.0\n1818,200.0\n1819,200.0\n1820,200.0\n1821,200.0\n1822,200.0\n1823,200.0\n1824,200.0\n1825,200.0\n1826,200.0\n1827,200.0\n1828,200.0\n1829,200.0\n1830,200.0\n1831,200.0\n1832,200.0\n1833,200.0\n1834,200.0\n1835,200.0\n1836,200.0\n1837,200.0\n1838,200.0\n1839,200.0\n1840,200.0\n1841,200.0\n1842,200.0\n1843,200.0\n1844,200.0\n1845,200.0\n1846,200.0\n1847,200.0\n1848,200.0\n1849,200.0\n1850,200.0\n1851,200.0\n1852,200.0\n1853,200.0\n1854,200.0\n1855,200.0\n1856,200.0\n1857,200.0\n1858,200.0\n1859,200.0\n1860,200.0\n1861,200.0\n1862,200.0\n1863,195.0\n1864,191.0\n1865,196.0\n1866,193.0\n1867,187.0\n1868,182.0\n1869,197.0\n1870,197.0\n1871,200.0\n1872,200.0\n1873,200.0\n1874,200.0\n1875,200.0\n1876,200.0\n1877,200.0\n1878,200.0\n1879,200.0\n1880,200.0\n1881,200.0\n1882,200.0\n1883,200.0\n1884,200.0\n1885,200.0\n1886,200.0\n1887,200.0\n1888,200.0\n1889,200.0\n1890,198.0\n1891,200.0\n1892,182.0\n1893,200.0\n1894,200.0\n1895,196.0\n1896,200.0\n1897,200.0\n1898,197.0\n1899,182.0\n1900,200.0\n1901,187.0\n1902,166.0\n1903,185.0\n1904,166.0\n1905,181.0\n1906,156.0\n1907,153.0\n1908,155.0\n1909,159.0\n1910,173.0\n1911,159.0\n1912,153.0\n1913,156.0\n1914,146.0\n1915,162.0\n1916,145.0\n1917,164.0\n1918,157.0\n1919,143.0\n1920,166.0\n1921,133.0\n1922,174.0\n1923,174.0\n1924,157.0\n1925,174.0\n1926,194.0\n1927,172.0\n1928,190.0\n1929,180.0\n1930,196.0\n1931,182.0\n1932,178.0\n1933,182.0\n1934,200.0\n1935,187.0\n1936,195.0\n1937,180.0\n1938,185.0\n1939,183.0\n1940,200.0\n1941,197.0\n1942,200.0\n1943,199.0\n1944,200.0\n1945,200.0\n1946,200.0\n1947,200.0\n1948,200.0\n1949,200.0\n1950,194.0\n1951,200.0\n1952,199.0\n1953,199.0\n1954,199.0\n1955,190.0\n1956,200.0\n1957,200.0\n1958,200.0\n1959,200.0\n1960,200.0\n1961,200.0\n1962,191.0\n1963,200.0\n1964,200.0\n1965,200.0\n1966,200.0\n1967,200.0\n1968,192.0\n1969,190.0\n1970,190.0\n1971,200.0\n1972,200.0\n1973,200.0\n1974,200.0\n1975,200.0\n1976,200.0\n1977,200.0\n1978,200.0\n1979,200.0\n1980,200.0\n1981,200.0\n1982,200.0\n1983,200.0\n1984,200.0\n1985,200.0\n1986,200.0\n1987,200.0\n1988,200.0\n1989,200.0\n1990,200.0\n1991,200.0\n1992,200.0\n1993,200.0\n1994,198.0\n1995,200.0\n1996,200.0\n1997,200.0\n1998,200.0\n1999,200.0\n2000,200.0\n2001,200.0\n2002,200.0\n" }, { "path": "history/ac_continue.csv", "content": "Episode_reward,actor_loss,critic_loss,episode\n-128.90520572493745,-0.11816113442182541,0.218042254447937,0\n-173.51670990790785,-0.06733673810958862,0.0800720751285553,1\n-187.8171977312419,-0.002720972988754511,0.002852647565305233,2\n-173.67273616272772,-0.036070723086595535,0.04723271355032921,3\n-156.9649702397321,-0.09143099188804626,0.11979616433382034,4\n-169.8156995095101,-0.03845040500164032,0.03830729424953461,5\n-183.36885903413648,-0.004992833361029625,0.004520351532846689,6\n-177.1069876314097,-0.0031833588145673275,0.008521170355379581,7\n-175.49269381318348,0.0024499772116541862,0.004748632665723562,8\n-165.72158234375667,0.018952136859297752,0.03374912217259407,9\n-160.16273015033792,-0.0030132168903946877,0.02405666373670101,10\n-168.84337562314553,-0.01132543571293354,0.009634644724428654,11\n-165.95408313158416,-0.012529523111879826,0.009909496642649174,12\n-162.8677057475428,-0.015982067212462425,0.004808629862964153,13\n-163.5753816694468,-0.06084240600466728,0.004999194294214249,14\n-156.0625641940695,-0.03364159166812897,0.006716023664921522,15\n-121.32997046665385,-0.10144391655921936,0.30767059326171875,16\n-101.54557290068891,-0.05899958312511444,0.13214406371116638,17\n-121.41828099390946,-0.011773789301514626,0.15117333829402924,18\n-156.06130000605796,0.03307399898767471,0.02828930877149105,19\n-155.56864909637278,0.03242425620555878,0.020547766238451004,20\n-145.36199061768616,0.02054990455508232,0.04608447104692459,21\n-151.7792733380804,0.03428905829787254,0.03264655917882919,22\n-139.84881185297428,-0.0009240770596079528,0.04941253736615181,23\n-150.03480041550247,-0.07503605633974075,0.021758461371064186,24\n-140.70167880684505,0.007148387376219034,0.052074089646339417,25\n-159.85961878595552,-0.030138272792100906,0.0173195693641901,26\n-158.76315171915036,-0.053707435727119446,0.005946178920567036,27\n-154.52882069043915,-0.031325992196798325,0.02829182706773281,28\n-158.9817618372644,-0.027006058022379875,0.006326370406895876,29\n-160.32050107010363,-0.05012276768684387,0.003761639818549156,30\n-149.30135812385507,-0.040228523313999176,0.009980856440961361,31\n-145.03741825592965,-0.01341972965747118,0.032863400876522064,32\n-118.49762380847548,0.0034106364473700523,0.012959661893546581,33\n-133.83763953421212,-0.028365153819322586,0.011703753843903542,34\n-116.23202698261296,-0.045169100165367126,0.026461677625775337,35\n-117.61107859971733,-0.0268267635256052,0.062097977846860886,36\n-149.894905565406,0.005340625066310167,0.08046764135360718,37\n-154.074162769591,-0.011202293448150158,0.012075615115463734,38\n-134.36350054408567,0.01002783328294754,0.037375565618276596,39\n-150.85834169505216,-0.035754427313804626,0.018294362351298332,40\n-145.50118224184877,-0.014716806821525097,0.006025801412761211,41\n-150.63776323816188,0.020595764741301537,0.015893014147877693,42\n-130.09002062901482,-0.002018730156123638,0.02221151813864708,43\n-156.9060387154177,-0.04334793612360954,0.004678923636674881,44\n-145.99038874505132,-0.035547368228435516,0.0054659126326441765,45\n-119.65712979195246,-0.018048271536827087,0.020358659327030182,46\n-150.21738698114189,-0.029474914073944092,0.007732502184808254,47\n-109.82576633740165,-0.007172092795372009,0.01311739906668663,48\n-149.36941666089103,-0.035635992884635925,0.008552027866244316,49\n-138.89770085503423,-0.03686399757862091,0.009956286288797855,50\n-141.0037562123717,-0.03394555673003197,0.006923982407897711,51\n-101.21312218966172,-0.004798423033207655,0.03019476681947708,52\n-148.75453726331315,-0.010606023482978344,0.24707195162773132,53\n-138.3855535751284,-0.03625988960266113,0.09367780387401581,54\n-127.78297799326874,-0.03951246291399002,0.0284748338162899,55\n-130.51456405908363,-0.011563140898942947,0.009528623893857002,56\n-137.87126854927783,-0.018686706200242043,0.014867258258163929,57\n-127.42378756734031,-0.02356593683362007,0.0075768339447677135,58\n-125.70859625109861,-0.033339206129312515,0.009255921468138695,59\n-119.05359541477232,-0.04140903800725937,0.02614688314497471,60\n-129.84422357598675,-0.009087968617677689,0.015148225240409374,61\n-90.63923559661282,-0.00835389643907547,0.04817446693778038,62\n-81.54950736175697,-0.027849368751049042,0.044494982808828354,63\n-129.2466770564977,-0.021401122212409973,0.02429121732711792,64\n-148.95003472734732,-0.037785496562719345,0.006264501716941595,65\n-129.42403966697339,-0.02046314626932144,0.018860243260860443,66\n-138.06388137480945,-0.026568971574306488,0.008116014301776886,67\n-135.52334782635612,-0.005726654548197985,0.004426317289471626,68\n-117.42493596719875,-0.03409242257475853,0.01645597256720066,69\n-113.3602089177063,-0.0016682073473930359,0.012022992596030235,70\n-90.90984976171424,0.018260039389133453,0.05911561846733093,71\n-78.78985121004055,0.04589872434735298,0.09445640444755554,72\n-90.34137985465165,0.009522655978798866,0.05463223159313202,73\n-64.33954879790882,-0.0007424831273965538,0.04739539325237274,74\n-79.52557487511162,0.022538084536790848,0.0658935084939003,75\n-65.03050377511977,-0.039056695997714996,0.04370497167110443,76\n-95.78177422592981,-0.03153213858604431,0.05397535488009453,77\n-66.51977812711509,-0.04175734892487526,0.031710051000118256,78\n-111.13418462322932,-0.020386528223752975,0.04701628535985947,79\n-78.01147432118017,0.0029152496717870235,0.053450051695108414,80\n-78.7047083325445,-0.03163432702422142,0.03992544114589691,81\n-62.88949006177542,-0.004482750315219164,0.038229282945394516,82\n-51.76042527448,-0.012605268508195877,0.024886665865778923,83\n-90.737748473562,-0.01493663527071476,0.036765206605196,84\n-97.83977206942664,-0.032980840653181076,0.0251147598028183,85\n-149.32810571283954,-0.06770310550928116,0.021020108833909035,86\n-94.74706307003167,-0.01194031722843647,0.06443748623132706,87\n-95.46576808934827,-0.03786374256014824,0.019207943230867386,88\n-158.87631174076228,-0.033470023423433304,0.01177558396011591,89\n-115.16114572097554,-0.06218460202217102,0.03541203588247299,90\n-65.5239862169029,0.013167922385036945,0.04131739214062691,91\n-115.44170789305701,-0.026105327531695366,0.014657242223620415,92\n-109.37613756038552,-0.028369365260004997,0.016077904030680656,93\n-124.4126175601616,-0.015185249038040638,0.00830584205687046,94\n-155.03070477429873,-0.028593778610229492,0.006861711852252483,95\n-105.1723590170853,-0.0010148596484214067,0.051170479506254196,96\n-93.13523837206614,-0.031695228070020676,0.030322112143039703,97\n-92.0780088085127,-0.014456412754952908,0.06276308000087738,98\n-157.98159915337575,-0.03909457102417946,0.008158702403306961,99\n-52.2910126989272,-0.0045435321517288685,0.07484341412782669,100\n-124.02489238812552,-0.12134473770856857,0.285476416349411,101\n-150.0383145206508,0.015463005751371384,0.1962101012468338,102\n-124.79979473721735,0.029857315123081207,0.06587255001068115,103\n-151.93783510203392,0.01816432923078537,0.06004641577601433,104\n-153.46082939526346,0.025976231321692467,0.018087945878505707,105\n-153.0643940476726,0.041255105286836624,0.01522158831357956,106\n-147.3605646925618,0.022746717557311058,0.05003218725323677,107\n-144.73965920710717,0.027152249589562416,0.02863638661801815,108\n-135.96452760584776,0.024353479966521263,0.032396815717220306,109\n-118.73612073550285,0.00028955817106179893,0.06907561421394348,110\n-98.50319380358809,0.028907084837555885,0.07416780292987823,111\n-122.32031241415092,0.0018870711792260408,0.04241432249546051,112\n-52.68386323472808,0.06541673839092255,0.050833940505981445,113\n-148.17981183314419,-0.05534721538424492,0.038898516446352005,114\n-158.66410200129022,-0.03129463270306587,0.002405643230304122,115\n-109.0615272106022,-0.02128031477332115,0.05848981812596321,116\n-124.9109849556943,0.008534370921552181,0.03075779601931572,117\n-135.04347801407317,-0.025528687983751297,0.011244556866586208,118\n-145.22483747259355,-0.023046214133501053,0.007142215967178345,119\n-143.25356316301173,-0.042317528277635574,0.017895519733428955,120\n-144.6934264522957,-0.032824937254190445,0.021521756425499916,121\n-130.0159842576681,0.0014981436543166637,0.04432859271764755,122\n-115.66706610998655,-0.008848265744745731,0.05973004922270775,123\n-127.51272586099758,-0.03261607512831688,0.0314054861664772,124\n-155.72650282309945,-0.029671311378479004,0.005902679171413183,125\n-153.19235486690124,-0.033465296030044556,0.005418254528194666,126\n-132.82926528195182,-0.0034157789777964354,0.019089041277766228,127\n-146.7012104483071,-0.027191782370209694,0.00704195536673069,128\n-140.18775701636244,-0.025628168135881424,0.011274627409875393,129\n-125.38016013503915,-0.024169037118554115,0.02482154406607151,130\n-100.43522021625355,-0.07547062635421753,0.08369606733322144,131\n-139.93956609761557,-0.028063612058758736,0.007911020889878273,132\n-133.39955124544062,-0.03748668357729912,0.014851379208266735,133\n-139.6341326355783,-0.028383878991007805,0.0059127239510416985,134\n-119.90442020868416,-0.017330193892121315,0.018415279686450958,135\n-117.1909711630189,-0.01607094518840313,0.018679514527320862,136\n-125.2698031165435,-0.017590919509530067,0.008127118460834026,137\n-136.92165300838857,-0.02812233380973339,0.007300146389752626,138\n-125.59011403449709,-0.004780275281518698,0.008428655564785004,139\n-124.65287338796519,-0.005278449039906263,0.009849241003394127,140\n-80.67677047875728,0.0029387688264250755,0.05523877590894699,141\n-158.06881067430632,-0.009701714850962162,0.016260672360658646,142\n-149.26043385530247,-0.02822364866733551,0.005844067316502333,143\n-115.98356587708639,3.375232336111367e-05,0.048886489123106,144\n-124.4142707287005,-0.01206234935671091,0.04648931324481964,145\n-147.7335764325854,-0.04082320258021355,0.00920157227665186,146\n-98.63444284701457,-0.014958950690925121,0.03036716766655445,147\n-123.65553689394451,-0.019268976524472237,0.02408592775464058,148\n-153.73841100505433,-0.03331742435693741,0.01404921431094408,149\n-118.65057358780756,0.004243273753672838,0.042835019528865814,150\n-158.1663332257977,-0.0138799287378788,0.006569362711161375,151\n-114.56265420000565,-0.03637496381998062,0.038454726338386536,152\n-82.02326355042123,0.008951845578849316,0.1299847960472107,153\n-156.35294001753226,-0.010119437240064144,0.03095921501517296,154\n-98.56655363503455,-0.02011219412088394,0.05828256532549858,155\n-138.26449021936,-0.04676579311490059,0.108129121363163,156\n-130.08203116572392,0.002649549162015319,0.04069558158516884,157\n-149.93142965469266,0.026073450222611427,0.043405186384916306,158\n-160.31117219680013,-0.0008374816388823092,0.015347685664892197,159\n-149.59651444694634,0.05680752918124199,0.0535757914185524,160\n-150.9233505659622,0.03021392412483692,0.03609118610620499,161\n-144.2835540547254,0.023117385804653168,0.040417127311229706,162\n-159.88837204485108,-0.007153730373829603,0.00622953474521637,163\n-150.78474056341705,0.02053423970937729,0.04304128512740135,164\n-136.03372166221877,-0.02200733870267868,0.17049196362495422,165\n-148.18934802724675,-0.024558095261454582,0.04619152843952179,166\n-151.2847497371689,0.02149391919374466,0.03446880355477333,167\n-158.8350684159196,-0.005283558741211891,0.0014910970348864794,168\n-152.81144919778453,0.0007866831147111952,0.0452098548412323,169\n-140.9202068856542,0.002955421106889844,0.05157984793186188,170\n-140.54003674604775,0.032337453216314316,0.06610522419214249,171\n-144.1423798065869,0.01574571430683136,0.04251216724514961,172\n-133.41967355793548,0.036541473120450974,0.04532473534345627,173\n-152.39713689540835,0.011199031956493855,0.02551628090441227,174\n-145.5557626919585,-0.004346379544585943,0.11224283277988434,175\n-150.37809405557874,0.04738866910338402,0.10094867646694183,176\n-142.30471975630527,-0.027170171961188316,0.029905376955866814,177\n-148.77613646117803,0.02065015397965908,0.024793371558189392,178\n-156.30290763867671,-0.026349496096372604,0.013088230043649673,179\n-150.45698192218146,0.026822783052921295,0.03536970913410187,180\n-118.82531151047955,0.022746743634343147,0.036443475633859634,181\n-137.35086893312732,-0.0023798823822289705,0.031157419085502625,182\n-107.45004553709376,-0.022908443585038185,0.029892168939113617,183\n-141.27673984687462,0.003531284863129258,0.09818501770496368,184\n-138.3823978754845,-0.010183075442910194,0.04263695701956749,185\n-98.03933186449947,0.05567610636353493,0.10059705376625061,186\n-124.51735539755273,-0.06041458994150162,0.026982726529240608,187\n-65.66860354333137,0.0636170282959938,0.038291048258543015,188\n-81.84785284651494,-0.0020879709627479315,0.035468075424432755,189\n-52.113815340505624,0.03987911343574524,0.041829537600278854,190\n-73.14194619074667,-0.012664618901908398,0.047694891691207886,191\n-157.84153737489083,-0.023931138217449188,0.011039887554943562,192\n-80.41695721543438,-0.03668137639760971,0.04912807047367096,193\n-85.4224593415252,-0.0130471708253026,0.03700229525566101,194\n-39.04873012349948,0.01953401044011116,0.04419881850481033,195\n-113.38524057930654,-0.01590311899781227,0.053968679159879684,196\n-78.35406210471025,-0.006963891908526421,0.05457195267081261,197\n-121.99739670812546,-0.012811277061700821,0.04539956897497177,198\n-109.73169673919942,0.0026423779781907797,0.041475724428892136,199\n-105.02001858407387,-0.01930549181997776,0.03942437469959259,200\n-124.64346357242043,-0.02973216213285923,0.02683805488049984,201\n-158.97169203246614,-0.022996850311756134,0.005472598131746054,202\n-55.975501705172185,-0.03305701166391373,0.04367101565003395,203\n-90.95345455783719,-0.03988717496395111,0.06918631494045258,204\n-140.48699088464716,-0.0098182437941432,0.029868654906749725,205\n-102.67706295142676,-0.024660401046276093,0.08131998032331467,206\n-76.95502994402632,0.022504493594169617,0.07006444782018661,207\n-93.52566294763956,-0.02607918716967106,0.046455267816782,208\n-113.8081430001521,-0.03847579285502434,0.037268634885549545,209\n-100.53893622022973,-0.02706807851791382,0.04681858420372009,210\n-41.42783467805271,-0.020472832024097443,0.05476333945989609,211\n-124.60772300858447,0.005437870044261217,0.04161582887172699,212\n-157.0984480853069,-0.013465720228850842,0.004529261030256748,213\n-128.66427159515996,-0.05583783984184265,0.037003979086875916,214\n-128.98599452262732,-0.04729686677455902,0.023879023268818855,215\n-110.25429588691877,-0.09077896177768707,0.06066092476248741,216\n-120.11882058255617,-0.013750850223004818,0.036198850721120834,217\n-140.42162211761624,-0.02635546401143074,0.025956392288208008,218\n-146.66137230809053,-0.017644003033638,0.015038125216960907,219\n-40.011129997533196,0.04881119728088379,0.06652971357107162,220\n-117.20162421839083,-0.013320665806531906,0.055077072232961655,221\n-127.32575085866836,-0.018112163990736008,0.03256919980049133,222\n-152.32876644491492,-0.016217157244682312,0.01988878659904003,223\n-137.87318563261618,-0.11298264563083649,0.329019695520401,224\n-148.91628818201937,0.036543361842632294,0.07683397084474564,225\n-150.678245305479,0.05873740091919899,0.0547025091946125,226\n-150.77903437207556,0.011996149085462093,0.030146360397338867,227\n-133.71793128390883,0.0332992747426033,0.07143933326005936,228\n-140.42604292530626,0.0189902875572443,0.044806819409132004,229\n-142.4752806440557,-0.006941778585314751,0.035318773239851,230\n-150.03500688030834,0.009044052101671696,0.029355064034461975,231\n-139.04874548524862,-0.020447663962841034,0.044867534190416336,232\n-146.83845227992944,-0.0023876719642430544,0.0382225438952446,233\n-142.58388430205832,0.009198383428156376,0.04529896751046181,234\n-134.8745679635526,0.020278535783290863,0.05896954610943794,235\n-134.80503921758992,0.01638519950211048,0.06039910390973091,236\n-148.89170484945157,0.014368806034326553,0.04100581631064415,237\n-137.0309946765213,-0.0289583969861269,0.05337051302194595,238\n-145.73603818858214,0.011523835361003876,0.04102834686636925,239\n-142.9160895737153,0.022374635562300682,0.04053419083356857,240\n-91.52763971642094,0.024928206577897072,0.073421411216259,241\n-52.61703835772214,0.024831969290971756,0.035120945423841476,242\n-50.933602723057504,-0.016637394204735756,0.057265061885118484,243\n-64.66246557071912,-0.03596227616071701,0.036646172404289246,244\n-51.91344908009875,-0.009127596393227577,0.03635336831212044,245\n-78.8106063596794,-0.026722149923443794,0.05818133428692818,246\n-131.91771729448482,0.0029772853013128042,0.03929189592599869,247\n-26.229942542948073,0.03517855703830719,0.022852012887597084,248\n-106.0781607419431,-0.07640299946069717,0.10025783628225327,249\n-159.10366695161107,-0.03250008448958397,0.014282047748565674,250\n-64.57643400726433,0.02826712094247341,0.041412536054849625,251\n-78.71924932135101,-0.013313734903931618,0.03240490332245827,252\n-52.455020884949114,-0.013538490980863571,0.031189188361167908,253\n-74.81716925905822,-0.03611985594034195,0.05175074562430382,254\n-90.00617358260301,-0.08829410374164581,0.052084069699048996,255\n-67.90359650291975,-0.032218579202890396,0.03038950450718403,256\n-71.54257529738055,-0.019493194296956062,0.026491494849324226,257\n-77.79305191875986,-0.09359975159168243,0.028879983350634575,258\n-41.49660884716948,-0.007207376416772604,0.021043358370661736,259\n-65.29821711289533,-0.03426751866936684,0.04617427662014961,260\n-157.65190295536127,-0.024333275854587555,0.00833294726908207,261\n-95.50223871201005,-0.03168012201786041,0.036341384053230286,262\n-80.82715112753124,-0.03059590421617031,0.026487188413739204,263\n-82.49194259934572,-0.007053407374769449,0.03766786679625511,264\n-0.16988046325441633,-0.00017406203551217914,8.908264135243371e-07,265\n-0.13265456481323898,-7.714905950706452e-05,5.980605806144013e-07,266\n-0.12260173922119749,-6.537969056807924e-06,4.4672387389255164e-07,267\n-94.42173826314847,-0.04438967630267143,0.07521101087331772,268\n-65.01396785148204,-0.012792889960110188,0.05583586171269417,269\n-158.41705798588742,-0.01075009722262621,0.02430165559053421,270\n-0.2986386800790562,0.00046946509974077344,7.062428721837932e-06,271\n-65.76089619126442,-0.02874954231083393,0.04023776203393936,272\n-13.56078677965855,-0.027453714981675148,0.015596875920891762,273\n-161.90905609970693,-0.030940227210521698,0.016619842499494553,274\n-26.58366386983076,-0.02131103351712227,0.02678052894771099,275\n-26.4752039783739,-0.03461957350373268,0.0360800139605999,276\n-39.35447062916321,-0.00841829925775528,0.044284526258707047,277\n-160.14692196687014,-0.023141052573919296,0.007844404317438602,278\n-26.18780562604063,-0.007899651303887367,0.05480824410915375,279\n-122.13339309399962,-0.04018831253051758,0.040583446621894836,280\n-52.65396682666125,-0.004132509231567383,0.05999450385570526,281\n-108.8547800469949,-0.033220838755369186,0.04260016605257988,282\n-93.33330283918276,-0.030547283589839935,0.03975842520594597,283\n-149.2842393186756,-0.019427135586738586,0.018807558342814445,284\n-159.56881981160265,-0.03281332552433014,0.003732141340151429,285\n-105.62199130929832,-0.027130236849188805,0.028560424223542213,286\n-67.46272220247214,-0.02661566436290741,0.028932569548487663,287\n-0.289600935035014,-5.0242590077687055e-05,7.010929152784229e-07,288\n-96.34998885095644,-0.03182423487305641,0.059965528547763824,289\n-52.84627718045333,-0.028930911794304848,0.04108802229166031,290\n-81.12083525533589,-0.029006628319621086,0.03198597580194473,291\n-105.86589411579529,-0.08842560648918152,0.13883435726165771,292\n-79.51643513963737,0.0013816994614899158,0.053708624094724655,293\n-0.15596549991063124,0.028698373585939407,0.004330551251769066,294\n-53.91334812383687,-0.024194039404392242,0.0608505979180336,295\n-94.48546394439745,0.004499424714595079,0.11409537494182587,296\n-66.1720022593302,0.022924331948161125,0.03452332317829132,297\n-40.27193942788025,-0.010099074803292751,0.035563912242650986,298\n-73.53049550451779,-0.03727560117840767,0.06251254677772522,299\n-0.17802951888561805,0.0002583649766165763,2.2563531274499837e-06,300\n-82.8453160076496,-0.0720314309000969,0.16718555986881256,301\n-112.79071781686828,-0.07028577476739883,0.1087086871266365,302\n-92.31103873728016,0.005163075868040323,0.03942626714706421,303\n-92.19721804123651,-0.04383604973554611,0.06991662085056305,304\n-65.43294974819163,-0.044362831860780716,0.09413345158100128,305\n-71.78681755939895,-0.006908423732966185,0.044377729296684265,306\n-26.296312000984045,-0.003579825861379504,0.016468774527311325,307\n-39.78674380996396,0.032871704548597336,0.09292875975370407,308\n-78.4225844984884,-0.0322578065097332,0.10654942691326141,309\n-77.5764192774322,0.0029102445114403963,0.04250193014740944,310\n-93.10038842095759,-0.05228818953037262,0.09260357916355133,311\n-72.56638710204888,-0.006537484936416149,0.042125921696424484,312\n-65.67072043251908,-0.01822699047625065,0.03742736950516701,313\n-39.63048076479607,-0.03022678568959236,0.01932057924568653,314\n-81.57633903032676,-0.04046407714486122,0.08362016826868057,315\n-0.3999511242819041,-0.0012998484307900071,0.00812481064349413,316\n-26.34744457274779,0.015010826289653778,0.03120376169681549,317\n-65.62875216019172,-0.031625378876924515,0.05775437504053116,318\n-57.238158380741105,2.1634101358358748e-05,0.033826034516096115,319\n-13.03000449518145,-0.027791587635874748,0.024137942120432854,320\n-13.263086435151326,-0.0020798766054213047,0.015616859309375286,321\n-0.23980391378834404,2.036239038716303e-06,4.93409288537805e-07,322\n-39.59459072099135,-0.04871445521712303,0.038839951157569885,323\n-38.95194855057293,-0.016660051420331,0.05507650971412659,324\n-115.56423112914591,-0.030799351632595062,0.07477468252182007,325\n-67.72627046432298,-0.00073583098128438,0.05578972026705742,326\n-121.68831463286674,-0.03669098764657974,0.06691554933786392,327\n-136.0618572239932,-0.030397621914744377,0.06042831018567085,328\n-107.65609156854728,0.004998049698770046,0.07635364681482315,329\n-159.35191949126605,-0.006734650116413832,0.010695707984268665,330\n-144.23526723075355,-0.0032358583994209766,0.0397292859852314,331\n-13.177122139797524,0.017588257789611816,0.023249244317412376,332\n-140.13245220365462,0.022651666775345802,0.06364542245864868,333\n-0.6909078193530611,-0.015548703260719776,0.008620300330221653,334\n-0.5809063671931585,-0.0003603027726057917,0.0004928346024826169,335\n-148.67940388353225,0.025887709110975266,0.04290877282619476,336\n-123.7029926543708,-0.02257213369011879,0.09929020702838898,337\n-76.80095216500077,0.0729958564043045,0.07670199126005173,338\n-91.30652644635214,-0.016187427565455437,0.050154224038124084,339\n-104.24790503396417,-0.02484406903386116,0.028579819947481155,340\n-85.2093164910138,0.010722042061388493,0.030846664682030678,341\n-109.86321898178868,-0.02547280117869377,0.049173444509506226,342\n-106.66345431906547,-0.027276229113340378,0.06361830979585648,343\n-156.62251445733983,-0.02034415677189827,0.032879430800676346,344\n-117.34566581461227,-0.02215372957289219,0.04584536701440811,345\n-111.03049947283328,-0.13828106224536896,0.1959814429283142,346\n-77.25504380973877,0.014753359369933605,0.03880394622683525,347\n-79.25465550712157,-0.041737738996744156,0.06206793710589409,348\n-77.81159838833679,-0.035130929201841354,0.04375605657696724,349\n-66.2578901699322,-0.025602025911211967,0.04114961624145508,350\n-64.32109044745705,-0.0614108070731163,0.04912661015987396,351\n-119.47176354548931,0.00034532189602032304,0.13319894671440125,352\n-131.81428279485232,-0.007083946373313665,0.06341385096311569,353\n-114.13586775534007,0.006838161963969469,0.048227161169052124,354\n-122.84534563144177,0.00707246083766222,0.045787420123815536,355\n-89.3372111934452,0.004162793513387442,0.055781684815883636,356\n-77.455975007448,-0.03300943970680237,0.041458576917648315,357\n-78.24745691402164,-0.057438597083091736,0.034609824419021606,358\n-77.48534981371445,-0.04557276889681816,0.039204347878694534,359\n-147.35581241535473,-0.03260508552193642,0.01366596482694149,360\n-151.21882988111395,-0.029944317415356636,0.007624490186572075,361\n-153.76092886446543,-0.043216366320848465,0.006593412719666958,362\n-90.93758126837973,-0.041089847683906555,0.07904022932052612,363\n-80.71338951379921,0.006569467950612307,0.06912820041179657,364\n-53.56679842127747,-0.02716740593314171,0.06527707725763321,365\n-82.31395940161049,-0.04524749889969826,0.05573941394686699,366\n-65.31397054206496,-0.01116603147238493,0.03947030380368233,367\n-65.78585635219996,-0.010563625022768974,0.0441976822912693,368\n-69.25677853186463,-0.03414547070860863,0.06310441344976425,369\n-41.84801459207408,-0.09057784080505371,0.03561611846089363,370\n-123.60045274517752,-0.11348411440849304,0.07958002388477325,371\n-158.40042048595762,-0.0095997778698802,0.012439865618944168,372\n-134.7026191169718,-0.006412342190742493,0.0583699606359005,373\n-119.36949283629957,-0.006020812317728996,0.04879005253314972,374\n-51.502852638686875,0.030146537348628044,0.09234855324029922,375\n-72.38128515991072,0.011469412595033646,0.07139140367507935,376\n-109.13642483303863,-0.019533904269337654,0.037893518805503845,377\n-77.8612831556905,-0.018678924068808556,0.05064314231276512,378\n-149.10992153878846,-0.0236445851624012,0.023128606379032135,379\n-104.17486459455499,-0.018880674615502357,0.04035788029432297,380\n-92.10392493420534,-0.04708840698003769,0.04466293752193451,381\n-122.62916791256436,-0.007454319391399622,0.011400115676224232,382\n-80.27965560317581,-0.020442957058548927,0.029220538213849068,383\n-121.78133388695602,-0.025428686290979385,0.014057955704629421,384\n-108.4478983150611,-0.046888839453458786,0.022410845384001732,385\n-62.85378436789923,-0.03716282919049263,0.0240864809602499,386\n-143.90706364553427,-0.035480089485645294,0.016230376437306404,387\n-71.69290105336381,-0.04514560103416443,0.03984649106860161,388\n-148.2371221678891,-0.01757737807929516,0.021576261147856712,389\n-158.87379815520634,-0.04719541594386101,0.008522883988916874,390\n-157.49557214036702,-0.04774903133511543,0.00976511836051941,391\n-152.79098571848493,-0.03778228908777237,0.00956884864717722,392\n-110.80371241392486,-0.04646831378340721,0.03844833746552467,393\n-76.49365234890963,-0.058387432247400284,0.03881101682782173,394\n-26.00982273449468,-0.015101365745067596,0.01956385001540184,395\n-0.4574479528929952,3.942980038118549e-05,4.657225417759037e-06,396\n-139.76015879950236,-0.11630292981863022,0.1931106150150299,397\n-143.9893110638805,0.015646863728761673,0.10971914231777191,398\n-71.05169875612877,0.0429181307554245,0.05598609894514084,399\n-75.78554000225454,0.017098909243941307,0.07171717286109924,400\n-140.42934462040597,-0.04376155883073807,0.03222441300749779,401\n-119.27542543770997,0.001271026092581451,0.03260697051882744,402\n-88.32552833007593,-0.002761855721473694,0.04469187557697296,403\n-133.12700947132333,-0.036382466554641724,0.09155251830816269,404\n-103.9428827704972,0.010799353010952473,0.04444997385144234,405\n-159.53074449517473,-0.05543727055191994,0.009400471113622189,406\n-122.80377675255926,-0.017933443188667297,0.030141081660985947,407\n-112.6544836856402,-0.0321430042386055,0.03212879225611687,408\n-120.35623137096026,-0.033010054379701614,0.015599047765135765,409\n-109.35014813615788,-0.04912981018424034,0.023191170766949654,410\n-102.08778099257555,-0.03018512763082981,0.030210718512535095,411\n-135.6133411839989,-0.020171955227851868,0.012096576392650604,412\n-129.32534677738923,-0.019673597067594528,0.011444742791354656,413\n-119.09527408877132,-0.021501565352082253,0.016383672133088112,414\n-104.37976331428638,-0.03095945529639721,0.0268239788711071,415\n-116.69949880179288,-0.056258317083120346,0.06633973121643066,416\n-157.83978196077464,-0.030293058604002,0.011732975021004677,417\n-103.91988594742278,0.008761337026953697,0.03878147900104523,418\n-135.52987079593086,-0.03915306180715561,0.013181419111788273,419\n-143.76345583011621,-0.024639803916215897,0.01082430500537157,420\n-141.38191986892488,-0.02614441327750683,0.007252200972288847,421\n-131.6835091961132,-0.018774695694446564,0.01054925937205553,422\n-93.09948145434258,0.007620969321578741,0.04173766449093819,423\n-158.41255265351637,-0.028101211413741112,0.017536787316203117,424\n-98.84931397530259,0.00412228237837553,0.06125400960445404,425\n-90.37808834600935,0.004088857676833868,0.056085217744112015,426\n-125.37931915407749,-0.009870398789644241,0.013467431999742985,427\n-129.29451671929962,-0.023519637063145638,0.01100924238562584,428\n-109.88070472978701,-0.025471951812505722,0.03275856003165245,429\n-96.66830856070594,-0.00995543971657753,0.02680511772632599,430\n-148.91799233505898,-0.01761895976960659,0.012967939488589764,431\n-157.5145294219545,-0.03992370143532753,0.012409992516040802,432\n-159.71196201663983,-0.043164242058992386,0.0036352372262626886,433\n-136.83785317362901,-0.032877322286367416,0.025246532633900642,434\n-129.07677065876854,-0.002297729253768921,0.017571426928043365,435\n-128.337435706807,-0.029865972697734833,0.015932030975818634,436\n-138.65016528196912,-0.028669025748968124,0.008770613931119442,437\n-159.6835156547871,-0.028522877022624016,0.01177776139229536,438\n-150.90629179722072,-0.023535024374723434,0.010596318170428276,439\n-145.18788803917926,-0.03319789469242096,0.012282084673643112,440\n-159.83600583363466,-0.037466853857040405,0.00470575550571084,441\n-120.27080538895002,-0.025502696633338928,0.07311707735061646,442\n-132.89509175998526,-0.011780848726630211,0.016009604558348656,443\n-151.81425909888355,-0.02975558303296566,0.010802528820931911,444\n-158.46746130269918,-0.046469975262880325,0.006291951984167099,445\n-113.62467869144047,-0.009485301561653614,0.0542246550321579,446\n-127.36422947529496,-0.0037976971361786127,0.01990300416946411,447\n-121.91919779232603,-0.023358898237347603,0.027330031618475914,448\n-89.28235436575278,-0.012660696171224117,0.04656204208731651,449\n-148.30799913327232,-0.018588846549391747,0.021362729370594025,450\n-143.4543282129579,-0.037606529891490936,0.020675456151366234,451\n-114.32844215772508,-0.01934315636754036,0.04110048711299896,452\n-141.88129762616921,-0.02802802063524723,0.01503519807010889,453\n-129.2503161664671,-0.03021426685154438,0.017883753404021263,454\n-120.62319227316313,-0.017387397587299347,0.02414742484688759,455\n-117.30791885104891,-0.01613880693912506,0.02275879867374897,456\n-157.0457753747779,-0.022862570360302925,0.014477331191301346,457\n-107.9812245117222,-0.037923287600278854,0.04696270450949669,458\n-114.55609954596484,-0.025405315682291985,0.027229860424995422,459\n-105.027528237553,-0.027183957397937775,0.03568172827363014,460\n-105.30925318410367,-0.012113668955862522,0.03603820502758026,461\n-132.7898618630805,-0.016477437689900398,0.02315354347229004,462\n-140.47949891598554,-0.01893824152648449,0.01675788126885891,463\n-131.4289571880956,-0.02234606444835663,0.025300854817032814,464\n-115.68304760435333,-0.011969472281634808,0.0318298414349556,465\n-143.68007278172726,-0.03444080427289009,0.01601318269968033,466\n-102.93201764920735,-0.0030736643821001053,0.05537399277091026,467\n-120.08156888409654,-0.004310398828238249,0.026737062260508537,468\n-158.3828158322219,-0.022283172234892845,0.017530949786305428,469\n-130.9686053926808,-0.09679147601127625,0.19618667662143707,470\n-104.43565710843941,0.023985719308257103,0.054824113845825195,471\n-150.8319972272468,-0.03582534193992615,0.015082096680998802,472\n-124.29911093365494,-0.003401998896151781,0.02254103682935238,473\n-120.77224410606729,-0.006498170550912619,0.0240518469363451,474\n-126.46661994865838,-0.03479928523302078,0.02397235482931137,475\n-139.1149242807633,-0.01937275566160679,0.009717773646116257,476\n-142.546401936936,-0.021950852125883102,0.010231674648821354,477\n-95.85089912604417,-0.00548981549218297,0.048925403505563736,478\n-107.85296023783714,0.0016318452544510365,0.030028043314814568,479\n-119.03653496800409,-0.019679594784975052,0.023646151646971703,480\n-159.28282492252322,-0.04855602607131004,0.01901816762983799,481\n-113.05835279564188,-0.0344134196639061,0.059326328337192535,482\n-140.5039039038542,-0.03065495938062668,0.013328935950994492,483\n-140.53442508041235,-0.021846134215593338,0.01192952785640955,484\n-137.2827547099211,-0.015718191862106323,0.018312832340598106,485\n-117.58628768991242,-0.044932566583156586,0.03964213281869888,486\n-121.25903569402358,-0.03715162351727486,0.027324898168444633,487\n-129.6133254285075,-0.032355472445487976,0.01864064671099186,488\n-159.65994531790597,-0.05241500958800316,0.012681905180215836,489\n-126.16926321199202,-0.012690901756286621,0.0644230991601944,490\n-135.90973930217606,-0.0094911465421319,0.019452156499028206,491\n-136.8541681022612,-0.01824874058365822,0.019122276455163956,492\n-137.4035503217135,-0.014235062524676323,0.023464588448405266,493\n-136.47030312281902,-0.019507575780153275,0.01940569467842579,494\n-139.8298566668537,-0.026058034971356392,0.011374448426067829,495\n-140.15553100932942,-0.01734858565032482,0.013210110366344452,496\n-128.5929624367314,-0.047601234167814255,0.02908635325729847,497\n-107.43755655811081,-0.0371660552918911,0.06429874151945114,498\n-97.75083918907556,-0.017936645075678825,0.04431864619255066,499\n" }, { "path": "history/ac_sparse.csv", "content": "Episode_reward,actor_loss,critic_loss,episode\n46.0,0.15708111226558685,10.703339576721191,0\n11.0,-0.28331494331359863,19.787118911743164,1\n24.0,0.07914095371961594,2.0553085803985596,2\n11.0,-0.16641865670681,6.423779487609863,3\n14.0,0.00206198007799685,3.0836663246154785,4\n24.0,-0.007437457796186209,1.3668951988220215,5\n15.0,0.07684779167175293,1.3325746059417725,6\n10.0,-0.14787694811820984,1.9680309295654297,7\n11.0,-0.5541443228721619,11.497489929199219,8\n15.0,0.046119723469018936,1.164766788482666,9\n27.0,0.18693281710147858,0.9977313280105591,10\n25.0,-0.02487783506512642,3.426077365875244,11\n15.0,0.09097448736429214,1.7190871238708496,12\n12.0,-0.11516990512609482,1.7416564226150513,13\n13.0,0.11680031567811966,0.9986221790313721,14\n11.0,-0.21710120141506195,1.3468047380447388,15\n18.0,-0.1584535390138626,6.159326076507568,16\n49.0,-0.02433660998940468,2.5991287231445312,17\n18.0,-0.016491122543811798,9.629115104675293,18\n27.0,0.1658116728067398,1.2685333490371704,19\n16.0,0.09101071953773499,0.8594657778739929,20\n11.0,-0.11546719819307327,3.541494846343994,21\n21.0,0.09801877290010452,1.6491456031799316,22\n10.0,0.05944998189806938,1.2436825037002563,23\n49.0,0.08347591012716293,1.292287826538086,24\n23.0,-0.10531073063611984,1.7366694211959839,25\n25.0,0.042967263609170914,1.4112411737442017,26\n14.0,-0.1030559092760086,0.5424199104309082,27\n13.0,0.09419719129800797,0.5872083902359009,28\n11.0,0.0714622437953949,0.25870734453201294,29\n28.0,-0.11955034732818604,3.0150628089904785,30\n23.0,0.12160929292440414,1.2548524141311646,31\n15.0,0.022263940423727036,1.052060842514038,32\n23.0,0.13196411728858948,0.8963538408279419,33\n15.0,-0.05559730529785156,0.8238027691841125,34\n15.0,0.03702358528971672,0.7466095089912415,35\n27.0,0.049668923020362854,1.0461552143096924,36\n29.0,0.007565909530967474,2.3898000717163086,37\n12.0,0.010593235492706299,0.8240801692008972,38\n57.0,0.06253640353679657,0.4960484504699707,39\n14.0,-0.11316702514886856,0.6910349726676941,40\n36.0,-0.05576689541339874,1.8883126974105835,41\n31.0,-0.06678037345409393,0.7510959506034851,42\n11.0,0.12745854258537292,0.9076582193374634,43\n26.0,0.08652591705322266,0.9330106973648071,44\n39.0,0.0007638564566150308,1.2670847177505493,45\n14.0,-0.1532670110464096,2.0817651748657227,46\n15.0,-0.18648196756839752,2.0914254188537598,47\n58.0,0.02371405065059662,0.874768853187561,48\n16.0,-0.1611773818731308,3.048701763153076,49\n78.0,-0.08101316541433334,0.7750633358955383,50\n108.0,0.00018569054373074323,0.6708148121833801,51\n20.0,-0.04658918455243111,1.9875085353851318,52\n10.0,0.15146510303020477,0.538233757019043,53\n51.0,0.07100915163755417,0.3324044644832611,54\n14.0,-0.21520325541496277,1.1971681118011475,55\n65.0,-0.06360554695129395,0.9158459305763245,56\n102.0,-0.007419256493449211,1.1813948154449463,57\n22.0,0.28063786029815674,0.49648362398147583,58\n99.0,0.059428099542856216,0.7366203665733337,59\n13.0,-0.26603060960769653,1.712687373161316,60\n28.0,0.055799055844545364,1.30489182472229,61\n28.0,-0.03550821915268898,1.184736967086792,62\n40.0,-0.1208341121673584,1.2666234970092773,63\n50.0,-0.010878819972276688,0.788481593132019,64\n102.0,-0.07855198532342911,0.6961331963539124,65\n78.0,-0.002004120498895645,1.04771888256073,66\n151.0,-0.01021483726799488,0.19554193317890167,67\n63.0,-0.017089037224650383,0.19626514613628387,68\n87.0,-0.036519408226013184,0.24972231686115265,69\n34.0,-0.08846504986286163,2.332099437713623,70\n24.0,0.09107737988233566,1.4093512296676636,71\n37.0,0.028210144490003586,1.1828194856643677,72\n35.0,0.03814493492245674,0.9228385090827942,73\n26.0,-0.1467811018228531,0.7876605987548828,74\n23.0,-0.19650426506996155,0.4523639678955078,75\n35.0,-0.12570612132549286,1.0840811729431152,76\n140.0,-0.07352928072214127,0.5347028970718384,77\n62.0,0.041730135679244995,0.7461478114128113,78\n28.0,-0.11057548969984055,1.640944242477417,79\n13.0,-0.28680145740509033,1.413116216659546,80\n46.0,-0.09727904945611954,0.8366326093673706,81\n105.0,0.006618705112487078,0.1780693084001541,82\n135.0,-0.07727474719285965,0.38812872767448425,83\n111.0,-0.04614066705107689,0.49178263545036316,84\n180.0,-0.027585316449403763,0.2371954768896103,85\n124.0,-0.032976407557725906,0.387373149394989,86\n127.0,-0.0004946341505274177,0.08610803633928299,87\n87.0,-0.05440782010555267,0.5575460195541382,88\n184.0,-0.02407199889421463,0.1114669069647789,89\n198.0,-0.02366279438138008,0.09785211086273193,90\n106.0,-0.04604828730225563,0.36168915033340454,91\n200.0,-0.015499483793973923,0.28062817454338074,92\n129.0,0.022404268383979797,0.34456419944763184,93\n200.0,-0.041753362864255905,0.17765121161937714,94\n200.0,-0.07258912920951843,0.3294238746166229,95\n200.0,0.0015863787848502398,0.3673955202102661,96\n59.0,-0.15577536821365356,0.9091365337371826,97\n123.0,-0.011310129426419735,0.36533528566360474,98\n76.0,-0.05190511420369148,0.20350292325019836,99\n142.0,-0.06325507164001465,0.34562650322914124,100\n138.0,-0.0152218546718359,0.39639076590538025,101\n108.0,-0.050525687634944916,0.520297110080719,102\n120.0,-0.014246219769120216,0.38258811831474304,103\n22.0,-0.13176153600215912,3.684641122817993,104\n23.0,0.07973098009824753,1.6745266914367676,105\n18.0,-0.13841216266155243,1.9822711944580078,106\n29.0,-0.22570228576660156,1.7687430381774902,107\n19.0,-0.03758680075407028,2.131235122680664,108\n24.0,0.1342272162437439,1.9011210203170776,109\n22.0,0.19661901891231537,2.4401755332946777,110\n21.0,-0.056327104568481445,1.658852219581604,111\n18.0,-0.35678890347480774,2.365894317626953,112\n19.0,0.030785605311393738,1.9986851215362549,113\n22.0,0.04046151787042618,1.9621682167053223,114\n14.0,-0.10542023926973343,2.52258038520813,115\n15.0,-0.19289381802082062,1.866510272026062,116\n19.0,-0.07956085354089737,1.8327311277389526,117\n12.0,-0.4135580062866211,2.4335200786590576,118\n11.0,-0.030774343758821487,0.5609723329544067,119\n13.0,0.035763032734394073,0.9815815687179565,120\n13.0,0.005579260643571615,1.1195242404937744,121\n14.0,-0.06337376683950424,1.74441397190094,122\n16.0,-0.0284061711281538,1.287758708000183,123\n17.0,-0.23556436598300934,1.4078991413116455,124\n17.0,-0.17262782156467438,0.38084855675697327,125\n16.0,-0.05455147475004196,0.7156497836112976,126\n15.0,-0.2710554897785187,0.6604633927345276,127\n27.0,-0.10968874394893646,1.159364938735962,128\n23.0,-0.10418415069580078,0.6072282791137695,129\n27.0,-0.08909005671739578,0.46529078483581543,130\n14.0,-0.42565593123435974,0.6341150403022766,131\n43.0,-0.03900247439742088,0.6616441607475281,132\n30.0,-0.13474325835704803,0.7914584279060364,133\n139.0,-0.14631320536136627,0.7484360933303833,134\n29.0,-0.05782928317785263,1.3499945402145386,135\n134.0,-0.056201860308647156,0.6385849118232727,136\n121.0,-0.011597949080169201,0.551180362701416,137\n121.0,-0.006616365630179644,0.3687339425086975,138\n200.0,-0.0016885180957615376,0.5285871028900146,139\n200.0,0.01933862827718258,0.49519073963165283,140\n117.0,0.025889186188578606,0.6685908436775208,141\n200.0,0.013803060166537762,0.5013015270233154,142\n118.0,0.03277798369526863,0.3231470286846161,143\n110.0,0.02277827262878418,0.25937753915786743,144\n199.0,-0.03243868425488472,0.11418214440345764,145\n32.0,0.0009617768228054047,3.5205748081207275,146\n20.0,0.2582176625728607,1.6396430730819702,147\n92.0,0.00787483062595129,0.1949927657842636,148\n29.0,-0.16344903409481049,3.3004753589630127,149\n20.0,0.10707885026931763,2.134247303009033,150\n15.0,-0.17798778414726257,2.2475526332855225,151\n15.0,-0.14601756632328033,2.5585546493530273,152\n16.0,-0.33845874667167664,2.1405718326568604,153\n24.0,-0.2621995508670807,1.6703158617019653,154\n13.0,0.139850914478302,2.7419891357421875,155\n20.0,-0.286944717168808,1.7754749059677124,156\n21.0,-0.17014659941196442,1.8928807973861694,157\n18.0,-0.4594283103942871,2.2065176963806152,158\n25.0,0.07775726169347763,1.7216529846191406,159\n25.0,-0.0004685783351305872,1.811466097831726,160\n26.0,-0.07042773813009262,1.722166657447815,161\n18.0,0.11300940066576004,2.355417490005493,162\n19.0,0.20305494964122772,2.207050085067749,163\n21.0,0.08928683400154114,1.6309477090835571,164\n21.0,0.10732945054769516,1.776181697845459,165\n20.0,0.08598322421312332,1.7819526195526123,166\n18.0,0.11096955835819244,1.8649431467056274,167\n21.0,0.12201087921857834,1.678673505783081,168\n24.0,0.019334256649017334,1.5406007766723633,169\n18.0,0.04419267177581787,2.0192480087280273,170\n18.0,-0.35479557514190674,1.848196268081665,171\n21.0,0.06767965108156204,1.5673719644546509,172\n27.0,0.048630241304636,1.358237385749817,173\n13.0,0.1493222862482071,2.587765693664551,174\n13.0,-0.1220095157623291,2.1480531692504883,175\n17.0,0.19160790741443634,1.5259854793548584,176\n19.0,-0.01122245006263256,1.432834267616272,177\n17.0,-0.0614984929561615,1.6416468620300293,178\n16.0,0.12375448644161224,1.671883463859558,179\n16.0,0.08760067075490952,1.731959342956543,180\n20.0,0.06143452972173691,1.328629493713379,181\n15.0,0.0514119453728199,1.7316960096359253,182\n15.0,-0.465212881565094,1.5180261135101318,183\n19.0,-0.009349101223051548,1.2548789978027344,184\n16.0,0.0371192991733551,1.4884580373764038,185\n17.0,0.04294077679514885,1.283705711364746,186\n17.0,-0.11588123440742493,1.42438542842865,187\n14.0,-0.046003036201000214,1.7050822973251343,188\n18.0,0.07881927490234375,1.1381348371505737,189\n20.0,0.05295684188604355,1.2035560607910156,190\n20.0,0.06974686682224274,1.0754977464675903,191\n22.0,0.04955059662461281,1.0104645490646362,192\n20.0,0.04513952136039734,0.9284613728523254,193\n27.0,0.06987472623586655,0.7489845752716064,194\n19.0,0.05672299116849899,0.8745599985122681,195\n23.0,0.012632045894861221,0.6619858741760254,196\n19.0,0.08306790143251419,0.5852164626121521,197\n24.0,-0.15606975555419922,0.5512746572494507,198\n25.0,-0.008989603258669376,0.5733245015144348,199\n22.0,-0.046394262462854385,0.6129249930381775,200\n29.0,0.005445077549666166,0.3126966655254364,201\n32.0,0.020901143550872803,0.16898809373378754,202\n33.0,0.0539248064160347,0.2498863935470581,203\n32.0,-0.022294219583272934,0.5785260796546936,204\n37.0,0.03526824340224266,0.2021041065454483,205\n33.0,-0.020538868382573128,0.40824177861213684,206\n97.0,-0.09483183920383453,0.7134478092193604,207\n38.0,0.061337921768426895,0.920579731464386,208\n111.0,-0.06470271199941635,0.3860238194465637,209\n40.0,0.029538234695792198,1.1659181118011475,210\n29.0,-0.37939706444740295,1.291352391242981,211\n32.0,0.03548508882522583,1.0397565364837646,212\n40.0,0.03945370391011238,0.8570400476455688,213\n33.0,0.02049228921532631,0.8838012218475342,214\n42.0,0.023618122562766075,0.7098052501678467,215\n106.0,-0.03726230189204216,0.6060228943824768,216\n117.0,0.071341373026371,0.2713908851146698,217\n117.0,-0.08965490758419037,0.364866703748703,218\n118.0,-0.05872584879398346,0.4178122878074646,219\n200.0,-0.02451316826045513,0.42843201756477356,220\n200.0,-0.042136091738939285,0.4635346233844757,221\n200.0,0.008176042698323727,0.42871609330177307,222\n151.0,0.00855812057852745,0.4203112721443176,223\n148.0,0.026913482695817947,0.5899422764778137,224\n33.0,-0.048606131225824356,1.609865665435791,225\n111.0,0.0280837994068861,0.3687347173690796,226\n141.0,-0.019896384328603745,0.2982586920261383,227\n103.0,0.01496926974505186,0.5740676522254944,228\n58.0,0.0004995541530661285,1.1529173851013184,229\n59.0,-0.06141338497400284,1.0899021625518799,230\n78.0,0.03473882004618645,0.3658626675605774,231\n54.0,-0.07987667620182037,1.1329048871994019,232\n66.0,0.016411136835813522,0.7592692375183105,233\n61.0,0.002788575366139412,0.5661239624023438,234\n55.0,-0.021031592041254044,0.41448068618774414,235\n82.0,-0.016050269827246666,0.15551184117794037,236\n68.0,-0.16944904625415802,0.6102570295333862,237\n116.0,0.020207425579428673,0.577196478843689,238\n137.0,-0.021703872829675674,0.682037889957428,239\n200.0,0.009493929333984852,0.4484395980834961,240\n200.0,0.023500967770814896,0.5558210611343384,241\n120.0,-0.0016839206218719482,0.5028929114341736,242\n116.0,0.006039291620254517,0.7781342267990112,243\n128.0,-0.013841862790286541,0.40437906980514526,244\n30.0,0.061143286526203156,2.245932102203369,245\n29.0,0.07970838248729706,1.4847197532653809,246\n24.0,0.03370336815714836,1.729762077331543,247\n38.0,0.039085377007722855,1.3021458387374878,248\n27.0,-0.09008951485157013,1.584365725517273,249\n25.0,0.03452019393444061,1.662104606628418,250\n16.0,0.040816254913806915,2.1942737102508545,251\n27.0,-0.0062103536911308765,1.2955496311187744,252\n17.0,-0.3921051621437073,1.918212652206421,253\n25.0,-0.1099080815911293,1.3688440322875977,254\n17.0,-0.20716571807861328,1.7599519491195679,255\n15.0,0.014386606402695179,1.8060944080352783,256\n18.0,0.03820337355136871,1.3439362049102783,257\n18.0,0.05200354754924774,1.298297643661499,258\n18.0,0.05704142153263092,1.2682714462280273,259\n18.0,0.043020494282245636,1.2042254209518433,260\n13.0,-0.002292733872309327,1.3885078430175781,261\n18.0,-0.10351388901472092,1.1175013780593872,262\n16.0,0.02520008012652397,1.2497649192810059,263\n24.0,0.10005620121955872,0.9423577785491943,264\n22.0,0.0005124238668940961,1.045102596282959,265\n16.0,-0.43298012018203735,1.250591516494751,266\n29.0,0.15496517717838287,0.7197147011756897,267\n25.0,0.02396712824702263,0.8432828783988953,268\n27.0,-0.04531298577785492,0.8296579718589783,269\n32.0,0.06497931480407715,0.593549370765686,270\n28.0,0.03618904948234558,0.5753760933876038,271\n43.0,0.06439494341611862,0.3444869816303253,272\n35.0,-0.03405074402689934,0.6553113460540771,273\n34.0,0.02783866412937641,0.3712359368801117,274\n48.0,0.018136590719223022,0.26566246151924133,275\n52.0,0.010908638127148151,0.23558005690574646,276\n34.0,0.017459429800510406,0.32277819514274597,277\n47.0,0.008845859207212925,0.29398858547210693,278\n47.0,-0.09002931416034698,0.3470979630947113,279\n64.0,0.03885677456855774,0.17960606515407562,280\n72.0,-0.00215338496491313,0.26457464694976807,281\n101.0,-0.032796621322631836,0.5474420189857483,282\n142.0,0.057146307080984116,0.12398028373718262,283\n198.0,-0.05464121326804161,0.47431960701942444,284\n200.0,-0.0029543363489210606,0.5001184344291687,285\n200.0,0.0036154913250356913,0.35193514823913574,286\n152.0,-0.007570174988359213,0.20036949217319489,287\n200.0,0.0020136034581810236,0.4522154927253723,288\n200.0,-0.021602289751172066,0.373005747795105,289\n120.0,0.004556996747851372,0.14547160267829895,290\n115.0,0.008248458616435528,0.07245007902383804,291\n114.0,0.007396819069981575,0.10600382089614868,292\n128.0,-0.033319175243377686,0.20940788090229034,293\n24.0,-0.10253982990980148,4.056668281555176,294\n24.0,0.19286853075027466,0.7121522426605225,295\n20.0,-0.05594431608915329,1.511155605316162,296\n25.0,-0.006276942323893309,1.1292327642440796,297\n39.0,0.03760023042559624,0.8104657530784607,298\n100.0,-0.05639946088194847,0.2665490508079529,299\n" }, { "path": "history/ddpg.csv", "content": "Episode_reward,Loss,episode\n-1509.9431573465693,9.224396642297506,0\n-1865.7656558462838,0.2480768134398386,1\n-1285.631963262328,0.9022780474415049,2\n-1305.4174017643334,2.977207661359571,3\n-1382.4000812392546,4.911441958639771,4\n-1062.575140308041,7.437465053331107,5\n-1463.8546658484654,10.082108230926096,6\n-1223.369614812585,15.84260846812278,7\n-952.641334606606,21.111351540870963,8\n-1188.393523377037,21.4787728536129,9\n-1336.629206109055,20.15197231501341,10\n-1526.5977045860482,27.844334089905022,11\n-1284.4889842946702,30.666733276881278,12\n-1602.4901520485619,33.38175935223698,13\n-1506.6540408292062,34.42411740764975,14\n-1388.615317798267,29.513498384170234,15\n-1283.3422825947046,44.503813500329855,16\n-1631.0748155793688,43.637542406357824,17\n-1218.7571524464163,38.54312503609806,18\n-1411.6802759192415,50.128491373732686,19\n-1200.9221829226055,49.92026059575379,20\n-1230.2951272159905,54.2671976512298,21\n-1115.6587490790553,53.428198878541586,22\n-1114.8459866674827,53.73677407834679,23\n-1547.5155195785896,63.951403808966276,24\n-1063.3868299522057,65.53570614930243,25\n-898.8103642220121,69.60633171230555,26\n-1772.604491489826,56.947230311110616,27\n-1184.3980703948423,56.36146053176373,28\n-759.2921595575442,56.576132576465604,29\n-659.4782017407694,56.29647995978594,30\n-861.2114618572491,58.280869005098936,31\n-868.5843914093462,60.638545464910564,32\n-1055.8006086159235,61.7349532661587,33\n-968.1073123743865,47.02249816864729,34\n-981.43887368482,54.137102286964655,35\n-761.0594396606434,34.231038222238425,36\n-1577.5114653357641,46.65215977922082,37\n-1180.6829526873669,41.02836825296283,38\n-624.3891328249243,45.36802312284708,39\n-627.0876307311714,42.32168261505663,40\n-1297.0037830418707,41.261543742641805,41\n-1322.6100932334898,35.52373902335763,42\n-849.5575525740325,31.8621961453557,43\n-1081.1936754523663,35.82060529649257,44\n-626.4815883735374,34.470040532425045,45\n-744.5436034922028,27.803453820869326,46\n-1463.0895354655895,37.31040966331959,47\n-858.4114601314827,23.049129573255776,48\n-1196.8466105893212,25.310880394652486,49\n-1221.4214918479825,34.453879231885075,50\n-635.5322105411383,29.389354399591685,51\n-630.4519218527109,28.666258409991862,52\n-1185.2536593137788,28.31704713396728,53\n-764.4381632336194,22.21421483851969,54\n-1313.2115576328438,24.51629433467984,55\n-858.4827687078472,32.937019220255316,56\n-741.389829745495,33.518929943814875,57\n-637.7492933687442,22.058465831577777,58\n-1565.3534979594217,23.153870964720845,59\n-1054.3859100346278,25.502650653645397,60\n-1770.917859770205,28.80989343907684,61\n-810.2497811602519,28.819833066910505,62\n-1057.2006563891528,26.13691723205149,63\n-1050.428922766867,24.413474108949302,64\n-874.1548399039127,28.675858917608856,65\n-1238.262404563758,32.64708211489022,66\n-1298.1190547629403,40.3973752079159,67\n-964.2918273021526,28.06319517850876,68\n-1235.3605769688743,29.70577700898051,69\n-1172.5004264568718,30.53858146555722,70\n-1168.3332468058768,34.81271961562336,71\n-1055.7145165992451,25.386454262509943,72\n-1003.6498049096768,40.83339177712798,73\n-1024.3221521298358,35.24092298142612,74\n-1042.4980651047051,41.469298264682294,75\n-1039.9132596264599,37.89584819301963,76\n-1017.8303287403377,34.35798047445714,77\n-987.178003404745,42.34256130099297,78\n-922.4054072981648,42.71066652163863,79\n-1086.5982370765053,37.53961860656738,80\n-1001.2568111933108,44.20261692024767,81\n-1039.9148148626332,39.62542889393866,82\n-1014.7862915770123,38.702078479006886,83\n-901.3799970078131,38.10230468176305,84\n-960.5569569363169,34.37458169274032,85\n-885.014240277523,42.73001478083432,86\n-878.3993387976839,35.681683946698904,87\n-870.4344527375974,42.747159790098664,88\n-1286.4680655815541,31.279732630178334,89\n-743.9724402397385,39.56026891417801,90\n-1440.5815302829,38.7252170599252,91\n-887.5124442350698,35.39897967994213,92\n-853.2025954302526,32.64205513432622,93\n-1107.243661383217,34.91410423785448,94\n-629.0496861884772,34.88018217563629,95\n-595.7645291736038,33.26384668022394,96\n-760.4614553011925,27.939529517665505,97\n-1012.3462343371923,28.840708961933853,98\n-878.0639777420679,30.402641356289386,99\n-880.5633393828522,24.96170378655195,100\n-1056.214777710658,23.492589546144007,101\n-893.3955082801795,25.264478589743376,102\n-763.1320657960539,24.1388235180825,103\n-505.50097175783213,23.13241209179163,104\n-553.7723736130598,26.654008081033826,105\n-742.5199257151457,26.6922037550807,106\n-961.3578975972842,23.324770770967007,107\n-795.0068184016638,26.934202596992254,108\n-753.3143419930043,26.45864893272519,109\n-749.7826401513026,23.12310247182846,110\n-1181.0581205410153,22.38455956786871,111\n-748.4255588221755,22.340058853551746,112\n-734.3846204042682,22.7055980373919,113\n-878.3373223390909,25.161819908469916,114\n-376.70457557936993,19.546717322915793,115\n-508.5545705869219,20.56222064897418,116\n-503.425602490802,17.00599652223289,117\n-375.91923117552994,24.69850500717759,118\n-249.37784986808146,18.53221425265074,119\n-1230.7925881337471,18.968866907656192,120\n-1047.3458950014876,19.263563225716354,121\n-520.5705180139792,19.088095309063792,122\n-127.41132204350131,16.680690984129907,123\n-494.0516561225265,14.114880212694407,124\n-241.7952363379058,15.163003908470273,125\n-622.1498929576501,13.860611055344343,126\n-126.28689389993978,10.743532688766718,127\n-481.7462220794856,12.154294685646892,128\n-252.0518702878864,8.22040078625083,129\n-1058.259944433226,10.099088204875589,130\n-1175.9008885759447,10.603743339329958,131\n-1380.0705786753408,10.230509576946497,132\n-778.3397305876816,15.391569518595935,133\n-133.46953696397125,14.409788343980908,134\n-900.7765687642241,15.92464128397405,135\n-865.3962979056647,12.270800278410315,136\n-895.7349032156494,9.966138138473035,137\n-637.3153610072421,14.353773091882466,138\n-643.3822494854851,12.138147334828973,139\n-529.7190572819877,11.232233720272779,140\n-267.4607062952199,11.33184901200235,141\n-381.90410426841044,8.22482607923448,142\n-131.40242751324504,7.3245576003938915,143\n-372.0020306586164,9.024972496032715,144\n-117.72238854186023,9.184057947322726,145\n-127.93529934947,9.165707747861743,146\n-649.3362683996553,12.506398611739279,147\n-124.71662549383197,9.226185390949249,148\n-125.02271326676247,10.296387278512121,149\n-2.9754935149676016,12.134869307279587,150\n-123.54315765522725,10.784451853185892,151\n-121.59970310844948,12.01533182270825,152\n-127.83470800816228,9.180831761807204,153\n-128.07100431495167,9.951997726485134,154\n-244.87853058154593,11.53757042095065,155\n-387.62244377800465,12.070866116993129,156\n-380.34783270446457,13.931512423604726,157\n-0.2049996884126917,17.255584732964635,158\n-0.1618008890243681,21.664329013191164,159\n-259.02778073541367,25.850017339736223,160\n-401.527916216881,20.795998985096812,161\n-119.79479164027957,22.34058747135103,162\n-1.099818418403321,20.18182430770248,163\n-369.6545190068,16.527924857279285,164\n-334.81240334087715,15.65822378879413,165\n-342.41405521558937,13.056955074761063,166\n-0.5857461587394992,11.401023681247606,167\n-260.2752873027791,9.589962152335794,168\n-261.66037029388485,7.553395981565118,169\n-129.56440024749517,5.8495552339963615,170\n-258.08730583463785,5.231778765879572,171\n-120.20548753691581,4.1807065053051335,172\n-260.95698845711877,3.291417717039585,173\n-389.37460725626335,2.9867228827252985,174\n-1187.125597823911,2.858089001998305,175\n-114.04855440980137,2.766956553393975,176\n-250.30580205457662,2.595750063546002,177\n-0.3014525534964593,2.9701660694740712,178\n-1493.5148444234912,2.094745456716046,179\n-388.5575184518914,3.2538931934162973,180\n-118.72923345772811,7.536445793844759,181\n-1130.9116601613175,5.056547564156354,182\n-118.80008688384058,7.5332822824828325,183\n-133.28311695911438,7.767288487832993,184\n-1128.0453572089707,7.902463755961508,185\n-4.918048537370808,9.419822578150779,186\n-507.91622762502357,13.655524237100035,187\n-116.65599103879406,11.857215057630093,188\n-119.57400403798566,12.931360413730145,189\n-126.60727857129913,10.384384954106062,190\n-130.69187463526168,11.276750937402248,191\n-532.6094861649789,9.913348460327834,192\n-119.40447754106185,10.843797551989555,193\n-118.01457640625543,15.639693554751576,194\n-125.38472296989859,14.797974425330757,195\n-0.7717357914055901,9.847735727513209,196\n-117.80949351393008,13.156731079723686,197\n-117.81753855241199,8.287331903707235,198\n-244.8041840890352,8.84490654880181,199\n" }, { "path": "history/ddqn.csv", "content": "Episode_reward,Loss,episode\n11.0,inf,0\n12.0,0.4681931138038635,5\n15.0,1.190276861190796,10\n12.0,1.9253158569335938,15\n14.0,2.1662869453430176,20\n14.0,0.3513646721839905,25\n13.0,1.5280985832214355,30\n9.0,7.641080379486084,35\n13.0,5.177434921264648,40\n37.0,0.32416990399360657,45\n30.0,3.0215706825256348,50\n36.0,2.573920726776123,55\n47.0,2.4598228931427,60\n47.0,0.20579954981803894,65\n59.0,0.9766868948936462,70\n114.0,0.021512869745492935,75\n78.0,0.020475244149565697,80\n141.0,0.018932819366455078,85\n200.0,0.31822460889816284,90\n198.0,0.03797835111618042,95\n191.0,0.014194854535162449,100\n182.0,0.013047108426690102,105\n176.0,2.1590566635131836,110\n155.0,0.4268774092197418,115\n157.0,0.024365421384572983,120\n150.0,0.010818028822541237,125\n143.0,0.6679432392120361,130\n137.0,0.012195337563753128,135\n139.0,0.016057107597589493,140\n133.0,0.006115872412919998,145\n130.0,0.011953135952353477,150\n139.0,0.00645385030657053,155\n147.0,0.014706989750266075,160\n150.0,0.03395920991897583,165\n132.0,0.006384558044373989,170\n147.0,0.0018864702433347702,175\n168.0,0.006037265993654728,180\n147.0,0.003880517091602087,185\n145.0,0.0685364305973053,190\n139.0,0.039683014154434204,195\n147.0,0.026838181540369987,200\n148.0,0.0033504890743643045,205\n153.0,0.027849620208144188,210\n148.0,0.0022808443754911423,215\n142.0,0.004483409691601992,220\n153.0,0.012921614572405815,225\n178.0,0.021095992997288704,230\n145.0,0.016173984855413437,235\n180.0,0.005053142085671425,240\n189.0,2.571658134460449,245\n200.0,0.005489872302860022,250\n200.0,0.003300785319879651,255\n200.0,0.010993970558047295,260\n200.0,0.02414834313094616,265\n200.0,0.0550992377102375,270\n200.0,0.028599431738257408,275\n200.0,0.04435380548238754,280\n200.0,0.07636132836341858,285\n200.0,0.02288103848695755,290\n161.0,0.026148678734898567,295\n182.0,0.046217601746320724,300\n200.0,0.09286589920520782,305\n200.0,0.027517562732100487,310\n200.0,0.03821772336959839,315\n200.0,0.013089239597320557,320\n200.0,5.8109049797058105,325\n11.0,0.007258770987391472,330\n200.0,0.09761332720518112,335\n200.0,3.223811388015747,340\n200.0,0.03628034517168999,345\n200.0,0.05087439715862274,350\n200.0,0.029234319925308228,355\n200.0,0.013562529347836971,360\n200.0,5.093896865844727,365\n200.0,0.015044460073113441,370\n185.0,0.012767013162374496,375\n160.0,4.742649555206299,380\n200.0,1.0178560018539429,385\n200.0,0.08182927221059799,390\n200.0,0.01739146187901497,395\n200.0,0.012645881623029709,400\n9.0,0.06882349401712418,405\n39.0,0.09852810204029083,410\n9.0,4.144737720489502,415\n200.0,0.14002403616905212,420\n200.0,1.0666762590408325,425\n200.0,3.7281572818756104,430\n200.0,0.03138595074415207,435\n200.0,1.6102722883224487,440\n200.0,0.039595987647771835,445\n200.0,0.02767912670969963,450\n183.0,0.012993409298360348,455\n179.0,1.8382511138916016,460\n200.0,0.01074942946434021,465\n119.0,0.05010325461626053,470\n125.0,0.04084702581167221,475\n177.0,0.01885252445936203,480\n167.0,0.5582287311553955,485\n158.0,0.33888524770736694,490\n200.0,0.03887341916561127,495\n191.0,0.5189582705497742,500\n200.0,4.817852973937988,505\n200.0,0.011416086927056313,510\n148.0,0.029537221416831017,515\n140.0,0.018659140914678574,520\n128.0,6.135469913482666,525\n140.0,0.017004529014229774,530\n141.0,0.07908019423484802,535\n154.0,0.014127008616924286,540\n136.0,0.11874083429574966,545\n143.0,0.05152087286114693,550\n200.0,0.010304479859769344,555\n200.0,0.7502464056015015,560\n157.0,0.06722593307495117,565\n200.0,0.018013518303632736,570\n200.0,5.614536762237549,575\n200.0,0.035216063261032104,580\n200.0,0.01069585606455803,585\n200.0,0.04107843339443207,590\n156.0,6.011622428894043,595\n" }, { "path": "history/dueling.csv", "content": "Episode_reward,Loss,episode\n30.0,inf,0\n12.0,0.47862666845321655,5\n10.0,0.5809901356697083,10\n8.0,1.5168644189834595,15\n10.0,6.394465446472168,20\n11.0,5.444173336029053,25\n11.0,6.27878999710083,30\n11.0,6.766901969909668,35\n10.0,0.267914354801178,40\n27.0,0.7681541442871094,45\n12.0,1.126452922821045,50\n21.0,2.1125664710998535,55\n13.0,1.1672120094299316,60\n12.0,0.19179925322532654,65\n28.0,1.5655790567398071,70\n37.0,3.6122264862060547,75\n25.0,0.4315088093280792,80\n44.0,3.3004751205444336,85\n54.0,3.665844440460205,90\n43.0,0.26683512330055237,95\n36.0,5.318463325500488,100\n103.0,3.0524964332580566,105\n50.0,1.7247264385223389,110\n22.0,9.666183471679688,115\n67.0,1.6773548126220703,120\n141.0,0.22905239462852478,125\n83.0,0.1352100670337677,130\n98.0,0.08889307826757431,135\n176.0,0.026787875220179558,140\n150.0,0.06743741035461426,145\n200.0,0.04945575073361397,150\n200.0,0.026455968618392944,155\n200.0,4.038379192352295,160\n200.0,0.051251500844955444,165\n200.0,0.018056631088256836,170\n200.0,6.713091850280762,175\n200.0,0.023181792348623276,180\n200.0,4.3332624435424805,185\n166.0,0.02410086989402771,190\n200.0,0.11829394102096558,195\n200.0,0.02812301553785801,200\n96.0,2.3146820068359375,205\n87.0,0.07561161369085312,210\n121.0,0.11904805898666382,215\n182.0,3.8535306453704834,220\n159.0,0.8626901507377625,225\n160.0,0.06850502640008926,230\n194.0,0.04625943303108215,235\n200.0,0.06545911729335785,240\n200.0,0.02540893666446209,245\n145.0,0.04876522347331047,250\n200.0,0.0599154457449913,255\n35.0,0.05168953537940979,260\n200.0,0.04558023437857628,265\n200.0,0.029182441532611847,270\n200.0,2.2100436687469482,275\n200.0,0.051673658192157745,280\n200.0,4.435661792755127,285\n200.0,0.043758101761341095,290\n200.0,0.03457247465848923,295\n200.0,0.019870392978191376,300\n200.0,0.019909145310521126,305\n9.0,0.1150365024805069,310\n200.0,0.07036980986595154,315\n71.0,0.05771002918481827,320\n200.0,0.09849052131175995,325\n200.0,0.045551616698503494,330\n137.0,0.06604130566120148,335\n11.0,0.09229154139757156,340\n200.0,3.8334453105926514,345\n200.0,0.02100004069507122,350\n171.0,0.02557738684117794,355\n200.0,5.395693302154541,360\n200.0,0.021021392196416855,365\n200.0,4.131258010864258,370\n200.0,0.03307604417204857,375\n172.0,0.059052661061286926,380\n200.0,0.04051003232598305,385\n200.0,0.018864136189222336,390\n200.0,0.05375618487596512,395\n39.0,0.0699274092912674,400\n200.0,1.667770266532898,405\n200.0,0.09974359720945358,410\n155.0,4.035706520080566,415\n157.0,0.07855170220136642,420\n200.0,0.028804700821638107,425\n162.0,1.9383234977722168,430\n155.0,0.02189595438539982,435\n200.0,0.014903474599123001,440\n200.0,0.07733364403247833,445\n187.0,0.022851835936307907,450\n178.0,0.018380625173449516,455\n131.0,0.04312446713447571,460\n11.0,0.017067477107048035,465\n200.0,0.07996176183223724,470\n200.0,0.6626131534576416,475\n200.0,0.019013769924640656,480\n126.0,0.011018311604857445,485\n185.0,0.0681515485048294,490\n170.0,0.022165827453136444,495\n200.0,0.025683466345071793,500\n141.0,1.391233205795288,505\n127.0,0.07900568842887878,510\n200.0,0.13426055014133453,515\n200.0,0.04910080507397652,520\n200.0,5.457238674163818,525\n116.0,0.046407174319028854,530\n125.0,0.05579424649477005,535\n176.0,0.07973217219114304,540\n199.0,0.037510018795728683,545\n142.0,0.04629438742995262,550\n12.0,0.15213751792907715,555\n10.0,0.025547455996274948,560\n35.0,0.16393166780471802,565\n196.0,0.10191859304904938,570\n187.0,0.060145795345306396,575\n188.0,0.028432875871658325,580\n181.0,1.3723012208938599,585\n200.0,0.035072945058345795,590\n140.0,4.833233833312988,595\n" }, { "path": "history/ndqn.csv", "content": "Episode_reward,Loss,episode\n12.0,inf,0\n39.0,0.5968765616416931,5\n10.0,0.16437163949012756,10\n8.0,0.8609060645103455,15\n10.0,4.647477149963379,20\n15.0,0.37436389923095703,25\n18.0,1.042937159538269,30\n32.0,0.25837475061416626,35\n16.0,1.1858779191970825,40\n29.0,0.13475389778614044,45\n36.0,0.8968307375907898,50\n75.0,0.09517502039670944,55\n66.0,5.1964287757873535,60\n169.0,0.8814995288848877,65\n200.0,0.04609198123216629,70\n200.0,0.07762521505355835,75\n98.0,1.7214298248291016,80\n66.0,0.02651367150247097,85\n48.0,2.0342352390289307,90\n89.0,0.04153415188193321,95\n65.0,0.013355078175663948,100\n90.0,0.026795051991939545,105\n72.0,0.021573634818196297,110\n106.0,0.02679624781012535,115\n110.0,0.010724630206823349,120\n200.0,0.9347834587097168,125\n200.0,2.6067371368408203,130\n200.0,0.03422386199235916,135\n173.0,0.023121735081076622,140\n74.0,0.03360900655388832,145\n107.0,0.0349336713552475,150\n105.0,3.056359052658081,155\n89.0,1.5287914276123047,160\n176.0,0.04428096488118172,165\n156.0,0.020566236227750778,170\n200.0,0.009313585236668587,175\n200.0,0.011389960534870625,180\n200.0,0.03430327773094177,185\n200.0,0.03263617306947708,190\n200.0,0.01683700643479824,195\n200.0,0.022440128028392792,200\n200.0,0.12438426911830902,205\n199.0,0.02578575909137726,210\n200.0,0.011934514157474041,215\n200.0,0.016942474991083145,220\n97.0,0.01568945124745369,225\n114.0,0.12535057961940765,230\n94.0,0.03555770218372345,235\n200.0,0.04525809735059738,240\n200.0,0.017689408734440804,245\n200.0,0.0067247869446873665,250\n200.0,0.07176659256219864,255\n200.0,0.08657388389110565,260\n141.0,0.033294644206762314,265\n121.0,0.028403853997588158,270\n200.0,2.290412664413452,275\n200.0,0.02531827799975872,280\n200.0,5.504024505615234,285\n178.0,0.08186638355255127,290\n200.0,0.08797675371170044,295\n200.0,0.020431581884622574,300\n132.0,5.576883316040039,305\n200.0,4.848743438720703,310\n200.0,0.016923055052757263,315\n200.0,0.02683131955564022,320\n124.0,0.015678368508815765,325\n177.0,0.03775625675916672,330\n137.0,0.05068105831742287,335\n140.0,0.40797001123428345,340\n166.0,0.02414533495903015,345\n155.0,0.01930900663137436,350\n164.0,0.011196689680218697,355\n176.0,0.004606300499290228,360\n167.0,0.0069297803565859795,365\n152.0,0.003465760499238968,370\n133.0,0.004011339973658323,375\n178.0,0.014116533100605011,380\n167.0,0.035929497331380844,385\n188.0,0.056661978363990784,390\n200.0,0.024751631543040276,395\n200.0,4.973880290985107,400\n168.0,0.09016193449497223,405\n131.0,0.05364896357059479,410\n143.0,0.03196289390325546,415\n175.0,0.010464230552315712,420\n137.0,0.043501902371644974,425\n155.0,0.037118226289749146,430\n124.0,0.013473874889314175,435\n145.0,0.03192755952477455,440\n200.0,0.02426156774163246,445\n122.0,0.026452424004673958,450\n162.0,3.928562641143799,455\n131.0,0.029229773208498955,460\n181.0,0.017442982643842697,465\n186.0,0.007039761170744896,470\n154.0,0.005024969577789307,475\n200.0,0.016365719959139824,480\n200.0,5.1473846435546875,485\n12.0,0.027667846530675888,490\n200.0,1.7594248056411743,495\n200.0,0.08335447311401367,500\n200.0,0.03457290679216385,505\n200.0,0.011456770822405815,510\n194.0,0.014004027470946312,515\n174.0,0.031052935868501663,520\n185.0,0.016657808795571327,525\n200.0,0.018321434035897255,530\n178.0,0.025208085775375366,535\n200.0,0.009419861249625683,540\n149.0,0.007532942574471235,545\n149.0,0.01075670588761568,550\n149.0,0.01898787170648575,555\n127.0,0.013626886531710625,560\n160.0,1.0664030313491821,565\n200.0,0.006966365966945887,570\n188.0,0.01090502180159092,575\n110.0,0.009709188714623451,580\n200.0,0.007233826443552971,585\n200.0,5.274221897125244,590\n200.0,0.058526113629341125,595\n" }, { "path": "history/pg.csv", "content": "Batch_reward,Episode_reward,Loss,episode\n126.0,25.2,-0.323914110660553,5\n170.0,34.0,-0.3671955466270447,10\n95.0,19.0,-0.31113868951797485,15\n79.0,15.8,-0.31862735748291016,20\n83.0,16.6,-0.31243768334388733,25\n74.0,14.8,-0.2790282964706421,30\n76.0,15.2,-0.30600470304489136,35\n69.0,13.8,-0.2797362208366394,40\n124.0,24.8,-0.31787288188934326,45\n94.0,18.8,-0.32114318013191223,50\n142.0,28.4,-0.3324996829032898,55\n110.0,22.0,-0.3379059135913849,60\n72.0,14.4,-0.31070083379745483,65\n160.0,32.0,-0.36996564269065857,70\n72.0,14.4,-0.3219853341579437,75\n80.0,16.0,-0.3177907168865204,80\n91.0,18.2,-0.31226593255996704,85\n115.0,23.0,-0.3390122056007385,90\n113.0,22.6,-0.32164397835731506,95\n178.0,35.6,-0.35060709714889526,100\n80.0,16.0,-0.3213381767272949,105\n116.0,23.2,-0.3130705952644348,110\n99.0,19.8,-0.3158109486103058,115\n102.0,20.4,-0.32293465733528137,120\n93.0,18.6,-0.30985644459724426,125\n117.0,23.4,-0.33105579018592834,130\n115.0,23.0,-0.3251442313194275,135\n113.0,22.6,-0.356834352016449,140\n126.0,25.2,-0.32076460123062134,145\n100.0,20.0,-0.3276306986808777,150\n89.0,17.8,-0.32530131936073303,155\n82.0,16.4,-0.3186661899089813,160\n128.0,25.6,-0.33558765053749084,165\n101.0,20.2,-0.31853118538856506,170\n108.0,21.6,-0.33060187101364136,175\n120.0,24.0,-0.32635703682899475,180\n154.0,30.8,-0.3299770951271057,185\n116.0,23.2,-0.3230454623699188,190\n197.0,39.4,-0.3595934212207794,195\n93.0,18.6,-0.3127687871456146,200\n133.0,26.6,-0.3340115547180176,205\n125.0,25.0,-0.32840797305107117,210\n99.0,19.8,-0.32053759694099426,215\n139.0,27.8,-0.3281051516532898,220\n141.0,28.2,-0.32602617144584656,225\n145.0,29.0,-0.33977046608924866,230\n96.0,19.2,-0.3226241171360016,235\n114.0,22.8,-0.33663344383239746,240\n111.0,22.2,-0.3363283574581146,245\n162.0,32.4,-0.3411697745323181,250\n128.0,25.6,-0.3296198844909668,255\n91.0,18.2,-0.32588136196136475,260\n111.0,22.2,-0.3307957649230957,265\n107.0,21.4,-0.3256392478942871,270\n129.0,25.8,-0.33310621976852417,275\n66.0,13.2,-0.3237151503562927,280\n121.0,24.2,-0.3433718979358673,285\n75.0,15.0,-0.3230298161506653,290\n117.0,23.4,-0.3261656165122986,295\n176.0,35.2,-0.3627626895904541,300\n121.0,24.2,-0.3326563835144043,305\n96.0,19.2,-0.32595303654670715,310\n98.0,19.6,-0.3163517415523529,315\n83.0,16.6,-0.3248583674430847,320\n123.0,24.6,-0.3384328782558441,325\n84.0,16.8,-0.3206236958503723,330\n122.0,24.4,-0.3618758022785187,335\n71.0,14.2,-0.32045409083366394,340\n102.0,20.4,-0.3334352970123291,345\n78.0,15.6,-0.3175198435783386,350\n154.0,30.8,-0.370138555765152,355\n107.0,21.4,-0.3431130051612854,360\n102.0,20.4,-0.3242567479610443,365\n91.0,18.2,-0.32735154032707214,370\n128.0,25.6,-0.33163779973983765,375\n89.0,17.8,-0.31738248467445374,380\n127.0,25.4,-0.38126614689826965,385\n78.0,15.6,-0.32667869329452515,390\n98.0,19.6,-0.3316572606563568,395\n147.0,29.4,-0.3489876985549927,400\n109.0,21.8,-0.328703373670578,405\n115.0,23.0,-0.3258844017982483,410\n77.0,15.4,-0.3242562413215637,415\n109.0,21.8,-0.3315679430961609,420\n125.0,25.0,-0.35489338636398315,425\n81.0,16.2,-0.32471272349357605,430\n78.0,15.6,-0.32005593180656433,435\n97.0,19.4,-0.31360965967178345,440\n125.0,25.0,-0.3440498113632202,445\n112.0,22.4,-0.33888572454452515,450\n128.0,25.6,-0.32566192746162415,455\n81.0,16.2,-0.325516015291214,460\n193.0,38.6,-0.3992687165737152,465\n159.0,31.8,-0.36642223596572876,470\n141.0,28.2,-0.3294885456562042,475\n115.0,23.0,-0.32550784945487976,480\n86.0,17.2,-0.32193079590797424,485\n117.0,23.4,-0.3378137946128845,490\n93.0,18.6,-0.3283882141113281,495\n91.0,18.2,-0.31916195154190063,500\n155.0,31.0,-0.35981062054634094,505\n122.0,24.4,-0.33529549837112427,510\n122.0,24.4,-0.3334076702594757,515\n127.0,25.4,-0.3241950273513794,520\n69.0,13.8,-0.31813812255859375,525\n100.0,20.0,-0.3260694742202759,530\n96.0,19.2,-0.3393671214580536,535\n108.0,21.6,-0.3339618146419525,540\n103.0,20.6,-0.32321697473526,545\n108.0,21.6,-0.34803831577301025,550\n209.0,41.8,-0.37500619888305664,555\n90.0,18.0,-0.31353095173835754,560\n91.0,18.2,-0.32434719800949097,565\n101.0,20.2,-0.3281025290489197,570\n169.0,33.8,-0.3593110740184784,575\n95.0,19.0,-0.331133633852005,580\n122.0,24.4,-0.3337944746017456,585\n135.0,27.0,-0.3440612852573395,590\n84.0,16.8,-0.3325881063938141,595\n101.0,20.2,-0.3345527946949005,600\n75.0,15.0,-0.3226890563964844,605\n111.0,22.2,-0.32936322689056396,610\n156.0,31.2,-0.34380999207496643,615\n89.0,17.8,-0.3252561688423157,620\n133.0,26.6,-0.3402594327926636,625\n106.0,21.2,-0.3349014222621918,630\n129.0,25.8,-0.33368977904319763,635\n108.0,21.6,-0.33249637484550476,640\n93.0,18.6,-0.3272244334220886,645\n116.0,23.2,-0.327767550945282,650\n94.0,18.8,-0.3070034980773926,655\n104.0,20.8,-0.32560011744499207,660\n89.0,17.8,-0.3445221483707428,665\n145.0,29.0,-0.3271236717700958,670\n110.0,22.0,-0.33397266268730164,675\n120.0,24.0,-0.32614800333976746,680\n123.0,24.6,-0.3399856388568878,685\n170.0,34.0,-0.36179420351982117,690\n192.0,38.4,-0.36600160598754883,695\n128.0,25.6,-0.32878589630126953,700\n117.0,23.4,-0.341595321893692,705\n134.0,26.8,-0.317445307970047,710\n95.0,19.0,-0.333281010389328,715\n146.0,29.2,-0.3421921133995056,720\n126.0,25.2,-0.3426232933998108,725\n107.0,21.4,-0.33393988013267517,730\n104.0,20.8,-0.32600662112236023,735\n146.0,29.2,-0.33033308386802673,740\n89.0,17.8,-0.32442706823349,745\n166.0,33.2,-0.36430254578590393,750\n166.0,33.2,-0.33611834049224854,755\n111.0,22.2,-0.32691603899002075,760\n165.0,33.0,-0.3286210894584656,765\n131.0,26.2,-0.3302786946296692,770\n128.0,25.6,-0.34573444724082947,775\n141.0,28.2,-0.34006255865097046,780\n141.0,28.2,-0.32640042901039124,785\n144.0,28.8,-0.32620134949684143,790\n191.0,38.2,-0.357293039560318,795\n125.0,25.0,-0.32633158564567566,800\n143.0,28.6,-0.3263686001300812,805\n124.0,24.8,-0.323049932718277,810\n177.0,35.4,-0.3612987697124481,815\n95.0,19.0,-0.3241707980632782,820\n162.0,32.4,-0.3730015158653259,825\n99.0,19.8,-0.3270459771156311,830\n119.0,23.8,-0.3332597017288208,835\n114.0,22.8,-0.32190534472465515,840\n109.0,21.8,-0.33435577154159546,845\n97.0,19.4,-0.3109934329986572,850\n85.0,17.0,-0.3222292363643646,855\n110.0,22.0,-0.32931065559387207,860\n79.0,15.8,-0.3263776898384094,865\n126.0,25.2,-0.3366556465625763,870\n109.0,21.8,-0.3179785907268524,875\n127.0,25.4,-0.33851853013038635,880\n108.0,21.6,-0.33515265583992004,885\n62.0,12.4,-0.32123520970344543,890\n130.0,26.0,-0.3218475580215454,895\n135.0,27.0,-0.33280518651008606,900\n104.0,20.8,-0.34523093700408936,905\n145.0,29.0,-0.32937273383140564,910\n150.0,30.0,-0.3692808151245117,915\n97.0,19.4,-0.33158236742019653,920\n160.0,32.0,-0.33346542716026306,925\n108.0,21.6,-0.3280292749404907,930\n214.0,42.8,-0.3434847891330719,935\n124.0,24.8,-0.34609928727149963,940\n153.0,30.6,-0.3434329926967621,945\n105.0,21.0,-0.3302847146987915,950\n118.0,23.6,-0.3245857357978821,955\n129.0,25.8,-0.3407069146633148,960\n147.0,29.4,-0.33763107657432556,965\n183.0,36.6,-0.3729757070541382,970\n129.0,25.8,-0.3388650119304657,975\n178.0,35.6,-0.355676531791687,980\n142.0,28.4,-0.3570093512535095,985\n94.0,18.8,-0.32580772042274475,990\n216.0,43.2,-0.34840127825737,995\n139.0,27.8,-0.3191074728965759,1000\n97.0,19.4,-0.321834534406662,1005\n165.0,33.0,-0.3293624520301819,1010\n128.0,25.6,-0.31909996271133423,1015\n90.0,18.0,-0.3221004605293274,1020\n120.0,24.0,-0.3225758969783783,1025\n110.0,22.0,-0.3229179382324219,1030\n183.0,36.6,-0.3358568251132965,1035\n107.0,21.4,-0.3292447030544281,1040\n140.0,28.0,-0.32179105281829834,1045\n136.0,27.2,-0.32573065161705017,1050\n246.0,49.2,-0.345488578081131,1055\n117.0,23.4,-0.34006670117378235,1060\n257.0,51.4,-0.33879947662353516,1065\n169.0,33.8,-0.350941926240921,1070\n185.0,37.0,-0.3586171865463257,1075\n141.0,28.2,-0.3215785622596741,1080\n152.0,30.4,-0.3365849554538727,1085\n157.0,31.4,-0.3653704524040222,1090\n159.0,31.8,-0.33440902829170227,1095\n120.0,24.0,-0.34093326330184937,1100\n201.0,40.2,-0.33844923973083496,1105\n219.0,43.8,-0.35109400749206543,1110\n263.0,52.6,-0.3550150692462921,1115\n201.0,40.2,-0.3496033847332001,1120\n313.0,62.6,-0.33947136998176575,1125\n302.0,60.4,-0.3442603647708893,1130\n308.0,61.6,-0.33771267533302307,1135\n251.0,50.2,-0.3532044291496277,1140\n148.0,29.6,-0.30538347363471985,1145\n233.0,46.6,-0.3530670702457428,1150\n319.0,63.8,-0.3307492136955261,1155\n350.0,70.0,-0.34717267751693726,1160\n384.0,76.8,-0.32274261116981506,1165\n422.0,84.4,-0.32839006185531616,1170\n228.0,45.6,-0.30992454290390015,1175\n367.0,73.4,-0.33390071988105774,1180\n398.0,79.6,-0.3453659117221832,1185\n565.0,113.0,-0.301306813955307,1190\n490.0,98.0,-0.32132676243782043,1195\n512.0,102.4,-0.31597256660461426,1200\n365.0,73.0,-0.3505546748638153,1205\n433.0,86.6,-0.32052144408226013,1210\n498.0,99.6,-0.33124780654907227,1215\n336.0,67.2,-0.3218598961830139,1220\n311.0,62.2,-0.3413241505622864,1225\n321.0,64.2,-0.3447491526603699,1230\n187.0,37.4,-0.31791070103645325,1235\n214.0,42.8,-0.29995036125183105,1240\n209.0,41.8,-0.33326685428619385,1245\n185.0,37.0,-0.28255948424339294,1250\n271.0,54.2,-0.3443583846092224,1255\n236.0,47.2,-0.3281277120113373,1260\n186.0,37.2,-0.3085566759109497,1265\n169.0,33.8,-0.29311126470565796,1270\n174.0,34.8,-0.28989309072494507,1275\n128.0,25.6,-0.2745927572250366,1280\n156.0,31.2,-0.2774413526058197,1285\n141.0,28.2,-0.2932639718055725,1290\n181.0,36.2,-0.2723075747489929,1295\n190.0,38.0,-0.28218960762023926,1300\n198.0,39.6,-0.27305474877357483,1305\n161.0,32.2,-0.2835294008255005,1310\n137.0,27.4,-0.28676924109458923,1315\n185.0,37.0,-0.30793672800064087,1320\n138.0,27.6,-0.2708069086074829,1325\n139.0,27.8,-0.2733404040336609,1330\n169.0,33.8,-0.25757548213005066,1335\n194.0,38.8,-0.27253004908561707,1340\n188.0,37.6,-0.2716241776943207,1345\n254.0,50.8,-0.2872135043144226,1350\n275.0,55.0,-0.30489450693130493,1355\n314.0,62.8,-0.30115067958831787,1360\n259.0,51.8,-0.28657886385917664,1365\n274.0,54.8,-0.2947578728199005,1370\n315.0,63.0,-0.284370481967926,1375\n420.0,84.0,-0.28122013807296753,1380\n304.0,60.8,-0.29020482301712036,1385\n347.0,69.4,-0.2792486846446991,1390\n436.0,87.2,-0.2617129385471344,1395\n298.0,59.6,-0.27968278527259827,1400\n290.0,58.0,-0.2966199517250061,1405\n293.0,58.6,-0.2687513530254364,1410\n301.0,60.2,-0.2640783488750458,1415\n297.0,59.4,-0.2812418043613434,1420\n295.0,59.0,-0.2812604308128357,1425\n308.0,61.6,-0.2759893238544464,1430\n346.0,69.2,-0.30547887086868286,1435\n275.0,55.0,-0.26983094215393066,1440\n272.0,54.4,-0.24922586977481842,1445\n419.0,83.8,-0.2561200261116028,1450\n316.0,63.2,-0.2530679702758789,1455\n317.0,63.4,-0.2537248432636261,1460\n306.0,61.2,-0.2625858783721924,1465\n329.0,65.8,-0.266976922750473,1470\n326.0,65.2,-0.2646888196468353,1475\n428.0,85.6,-0.2437351644039154,1480\n316.0,63.2,-0.260174036026001,1485\n367.0,73.4,-0.27776938676834106,1490\n373.0,74.6,-0.26097598671913147,1495\n318.0,63.6,-0.27720341086387634,1500\n346.0,69.2,-0.2672405540943146,1505\n325.0,65.0,-0.2874023914337158,1510\n415.0,83.0,-0.25887230038642883,1515\n503.0,100.6,-0.26642149686813354,1520\n418.0,83.6,-0.26785600185394287,1525\n322.0,64.4,-0.2640377879142761,1530\n570.0,114.0,-0.24567648768424988,1535\n343.0,68.6,-0.26842209696769714,1540\n323.0,64.6,-0.27676114439964294,1545\n296.0,59.2,-0.2819293141365051,1550\n619.0,123.8,-0.26687514781951904,1555\n416.0,83.2,-0.284103661775589,1560\n498.0,99.6,-0.29981642961502075,1565\n268.0,53.6,-0.26020547747612,1570\n428.0,85.6,-0.2733575105667114,1575\n411.0,82.2,-0.29171478748321533,1580\n363.0,72.6,-0.2915985882282257,1585\n376.0,75.2,-0.2707919180393219,1590\n452.0,90.4,-0.2971310317516327,1595\n367.0,73.4,-0.2947559058666229,1600\n380.0,76.0,-0.2887576222419739,1605\n519.0,103.8,-0.2703001797199249,1610\n407.0,81.4,-0.2852098345756531,1615\n668.0,133.6,-0.2705146074295044,1620\n436.0,87.2,-0.30569276213645935,1625\n418.0,83.6,-0.2917002737522125,1630\n527.0,105.4,-0.2697019577026367,1635\n541.0,108.2,-0.2923319935798645,1640\n450.0,90.0,-0.2872810661792755,1645\n486.0,97.2,-0.26877787709236145,1650\n586.0,117.2,-0.2843734323978424,1655\n504.0,100.8,-0.29393646121025085,1660\n495.0,99.0,-0.28225526213645935,1665\n426.0,85.2,-0.31057173013687134,1670\n444.0,88.8,-0.31152743101119995,1675\n456.0,91.2,-0.28087612986564636,1680\n471.0,94.2,-0.2913546562194824,1685\n496.0,99.2,-0.2839125692844391,1690\n467.0,93.4,-0.2844887375831604,1695\n463.0,92.6,-0.27567774057388306,1700\n518.0,103.6,-0.27867573499679565,1705\n537.0,107.4,-0.28661832213401794,1710\n638.0,127.6,-0.2699316143989563,1715\n670.0,134.0,-0.2734288275241852,1720\n769.0,153.8,-0.26910433173179626,1725\n645.0,129.0,-0.26937440037727356,1730\n456.0,91.2,-0.2961300015449524,1735\n591.0,118.2,-0.27045905590057373,1740\n672.0,134.4,-0.26792922616004944,1745\n643.0,128.6,-0.28915852308273315,1750\n841.0,168.2,-0.25474753975868225,1755\n744.0,148.8,-0.26696857810020447,1760\n754.0,150.8,-0.2589016258716583,1765\n745.0,149.0,-0.27511686086654663,1770\n678.0,135.6,-0.27377811074256897,1775\n542.0,108.4,-0.28012797236442566,1780\n482.0,96.4,-0.2984004616737366,1785\n564.0,112.8,-0.28570491075515747,1790\n526.0,105.2,-0.28553274273872375,1795\n429.0,85.8,-0.31285083293914795,1800\n642.0,128.4,-0.28988713026046753,1805\n789.0,157.8,-0.25857824087142944,1810\n618.0,123.6,-0.30434682965278625,1815\n693.0,138.6,-0.27829989790916443,1820\n821.0,164.2,-0.26921334862709045,1825\n881.0,176.2,-0.24802248179912567,1830\n922.0,184.4,-0.25607961416244507,1835\n763.0,152.6,-0.2877015173435211,1840\n731.0,146.2,-0.27064964175224304,1845\n818.0,163.6,-0.26569467782974243,1850\n765.0,153.0,-0.27027228474617004,1855\n844.0,168.8,-0.2618628740310669,1860\n589.0,117.8,-0.29191330075263977,1865\n787.0,157.4,-0.2632910907268524,1870\n867.0,173.4,-0.25515371561050415,1875\n915.0,183.0,-0.24169787764549255,1880\n752.0,150.4,-0.2643362581729889,1885\n749.0,149.8,-0.27746638655662537,1890\n859.0,171.8,-0.2660120725631714,1895\n775.0,155.0,-0.27933937311172485,1900\n694.0,138.8,-0.2731143534183502,1905\n851.0,170.2,-0.26047879457473755,1910\n611.0,122.2,-0.3043682277202606,1915\n648.0,129.6,-0.2933899164199829,1920\n529.0,105.8,-0.2947007119655609,1925\n360.0,72.0,-0.3085305988788605,1930\n495.0,99.0,-0.32446032762527466,1935\n767.0,153.4,-0.26316165924072266,1940\n482.0,96.4,-0.29827943444252014,1945\n458.0,91.6,-0.3013209402561188,1950\n569.0,113.8,-0.2856493592262268,1955\n540.0,108.0,-0.2954277992248535,1960\n657.0,131.4,-0.2848345935344696,1965\n567.0,113.4,-0.2670009136199951,1970\n661.0,132.2,-0.2961806654930115,1975\n374.0,74.8,-0.31242892146110535,1980\n514.0,102.8,-0.29416441917419434,1985\n439.0,87.8,-0.3018152713775635,1990\n589.0,117.8,-0.2866000533103943,1995\n673.0,134.6,-0.27572011947631836,2000\n491.0,98.2,-0.28802257776260376,2005\n502.0,100.4,-0.2913190424442291,2010\n684.0,136.8,-0.2605753540992737,2015\n585.0,117.0,-0.2784154713153839,2020\n490.0,98.0,-0.3181760609149933,2025\n481.0,96.2,-0.3038674592971802,2030\n559.0,111.8,-0.2845531105995178,2035\n344.0,68.8,-0.3110370337963104,2040\n468.0,93.6,-0.32684192061424255,2045\n550.0,110.0,-0.2799729108810425,2050\n498.0,99.6,-0.2940945327281952,2055\n478.0,95.6,-0.3053005337715149,2060\n638.0,127.6,-0.2735568881034851,2065\n501.0,100.2,-0.32170242071151733,2070\n731.0,146.2,-0.2598632574081421,2075\n577.0,115.4,-0.27256399393081665,2080\n495.0,99.0,-0.2753492295742035,2085\n655.0,131.0,-0.2919852137565613,2090\n656.0,131.2,-0.27932503819465637,2095\n519.0,103.8,-0.28536275029182434,2100\n425.0,85.0,-0.32349893450737,2105\n303.0,60.6,-0.32709527015686035,2110\n464.0,92.8,-0.3068963289260864,2115\n354.0,70.8,-0.2968566119670868,2120\n459.0,91.8,-0.29264581203460693,2125\n454.0,90.8,-0.29909512400627136,2130\n480.0,96.0,-0.30375662446022034,2135\n511.0,102.2,-0.3016867935657501,2140\n438.0,87.6,-0.28403419256210327,2145\n325.0,65.0,-0.3250955045223236,2150\n445.0,89.0,-0.3128427267074585,2155\n476.0,95.2,-0.2751147747039795,2160\n349.0,69.8,-0.3309052586555481,2165\n323.0,64.6,-0.29476287961006165,2170\n306.0,61.2,-0.29730474948883057,2175\n275.0,55.0,-0.28807350993156433,2180\n262.0,52.4,-0.2849494516849518,2185\n247.0,49.4,-0.2895693778991699,2190\n390.0,78.0,-0.28957656025886536,2195\n398.0,79.6,-0.30953487753868103,2200\n315.0,63.0,-0.3229285478591919,2205\n244.0,48.8,-0.29578179121017456,2210\n232.0,46.4,-0.2965617775917053,2215\n325.0,65.0,-0.30256932973861694,2220\n217.0,43.4,-0.2804889678955078,2225\n286.0,57.2,-0.3051661550998688,2230\n366.0,73.2,-0.30820080637931824,2235\n250.0,50.0,-0.2960183918476105,2240\n387.0,77.4,-0.3190012574195862,2245\n349.0,69.8,-0.3200511932373047,2250\n313.0,62.6,-0.3188655376434326,2255\n320.0,64.0,-0.31194472312927246,2260\n372.0,74.4,-0.29605546593666077,2265\n392.0,78.4,-0.29973769187927246,2270\n381.0,76.2,-0.3098653554916382,2275\n307.0,61.4,-0.309661328792572,2280\n295.0,59.0,-0.3135487735271454,2285\n310.0,62.0,-0.32026296854019165,2290\n360.0,72.0,-0.3253321349620819,2295\n255.0,51.0,-0.3225225806236267,2300\n260.0,52.0,-0.28479400277137756,2305\n304.0,60.8,-0.30789539217948914,2310\n469.0,93.8,-0.3014775216579437,2315\n300.0,60.0,-0.3001944422721863,2320\n303.0,60.6,-0.280273973941803,2325\n325.0,65.0,-0.30136871337890625,2330\n323.0,64.6,-0.3079989552497864,2335\n392.0,78.4,-0.32181212306022644,2340\n325.0,65.0,-0.31631535291671753,2345\n515.0,103.0,-0.2901548147201538,2350\n370.0,74.0,-0.3226360082626343,2355\n268.0,53.6,-0.30735111236572266,2360\n408.0,81.6,-0.31560268998146057,2365\n338.0,67.6,-0.3099435269832611,2370\n282.0,56.4,-0.2960219085216522,2375\n276.0,55.2,-0.3091493546962738,2380\n400.0,80.0,-0.3270515203475952,2385\n383.0,76.6,-0.32039105892181396,2390\n298.0,59.6,-0.3318256437778473,2395\n403.0,80.6,-0.30560219287872314,2400\n389.0,77.8,-0.3165701627731323,2405\n314.0,62.8,-0.3352294862270355,2410\n327.0,65.4,-0.3150443136692047,2415\n440.0,88.0,-0.3004245162010193,2420\n437.0,87.4,-0.31933194398880005,2425\n377.0,75.4,-0.3215472400188446,2430\n449.0,89.8,-0.30864641070365906,2435\n283.0,56.6,-0.3230055272579193,2440\n363.0,72.6,-0.321520060300827,2445\n340.0,68.0,-0.33344605565071106,2450\n398.0,79.6,-0.3284223675727844,2455\n278.0,55.6,-0.3242235481739044,2460\n387.0,77.4,-0.3207733631134033,2465\n401.0,80.2,-0.3539959192276001,2470\n294.0,58.8,-0.3313872814178467,2475\n381.0,76.2,-0.3344331979751587,2480\n214.0,42.8,-0.3258454501628876,2485\n361.0,72.2,-0.30855175852775574,2490\n202.0,40.4,-0.3441172242164612,2495\n344.0,68.8,-0.3241690397262573,2500\n265.0,53.0,-0.3344844877719879,2505\n280.0,56.0,-0.3179953098297119,2510\n295.0,59.0,-0.33485040068626404,2515\n442.0,88.4,-0.33316734433174133,2520\n379.0,75.8,-0.35698387026786804,2525\n318.0,63.6,-0.33826303482055664,2530\n251.0,50.2,-0.3398612439632416,2535\n300.0,60.0,-0.32650598883628845,2540\n255.0,51.0,-0.33304497599601746,2545\n226.0,45.2,-0.33001911640167236,2550\n409.0,81.8,-0.3345124125480652,2555\n375.0,75.0,-0.3362683951854706,2560\n362.0,72.4,-0.3233446180820465,2565\n352.0,70.4,-0.33965399861335754,2570\n317.0,63.4,-0.35508760809898376,2575\n283.0,56.6,-0.34275469183921814,2580\n372.0,74.4,-0.3211212158203125,2585\n405.0,81.0,-0.3200519382953644,2590\n412.0,82.4,-0.3162257969379425,2595\n588.0,117.6,-0.2985230088233948,2600\n363.0,72.6,-0.3323720693588257,2605\n335.0,67.0,-0.3439211845397949,2610\n398.0,79.6,-0.3534368872642517,2615\n267.0,53.4,-0.3095570504665375,2620\n323.0,64.6,-0.32619574666023254,2625\n472.0,94.4,-0.3172033131122589,2630\n348.0,69.6,-0.3429124355316162,2635\n264.0,52.8,-0.2987551689147949,2640\n368.0,73.6,-0.35467034578323364,2645\n316.0,63.2,-0.3708944618701935,2650\n331.0,66.2,-0.3331873118877411,2655\n325.0,65.0,-0.35394689440727234,2660\n320.0,64.0,-0.3461282551288605,2665\n287.0,57.4,-0.3383663594722748,2670\n299.0,59.8,-0.34119999408721924,2675\n358.0,71.6,-0.3390156924724579,2680\n315.0,63.0,-0.3429752290248871,2685\n170.0,34.0,-0.3075783848762512,2690\n333.0,66.6,-0.33995160460472107,2695\n321.0,64.2,-0.35553258657455444,2700\n292.0,58.4,-0.32117390632629395,2705\n384.0,76.8,-0.3236222267150879,2710\n382.0,76.4,-0.3598511815071106,2715\n299.0,59.8,-0.3402957320213318,2720\n440.0,88.0,-0.33124813437461853,2725\n345.0,69.0,-0.3368532955646515,2730\n401.0,80.2,-0.3405100107192993,2735\n373.0,74.6,-0.35578832030296326,2740\n304.0,60.8,-0.3320891857147217,2745\n409.0,81.8,-0.35458457469940186,2750\n479.0,95.8,-0.3229633867740631,2755\n332.0,66.4,-0.37328967452049255,2760\n345.0,69.0,-0.3245355188846588,2765\n416.0,83.2,-0.3375442326068878,2770\n273.0,54.6,-0.3345900774002075,2775\n523.0,104.6,-0.3130764961242676,2780\n349.0,69.8,-0.34857919812202454,2785\n246.0,49.2,-0.3524782657623291,2790\n261.0,52.2,-0.3142869770526886,2795\n281.0,56.2,-0.3461417257785797,2800\n410.0,82.0,-0.32556286454200745,2805\n445.0,89.0,-0.314216673374176,2810\n427.0,85.4,-0.3143163025379181,2815\n493.0,98.6,-0.32997140288352966,2820\n288.0,57.6,-0.3426615595817566,2825\n252.0,50.4,-0.32277217507362366,2830\n297.0,59.4,-0.3320043683052063,2835\n339.0,67.8,-0.3409106731414795,2840\n372.0,74.4,-0.34068194031715393,2845\n281.0,56.2,-0.34562399983406067,2850\n240.0,48.0,-0.3473271429538727,2855\n348.0,69.6,-0.3337392210960388,2860\n318.0,63.6,-0.3524182140827179,2865\n214.0,42.8,-0.32239580154418945,2870\n341.0,68.2,-0.3237490952014923,2875\n212.0,42.4,-0.34856706857681274,2880\n308.0,61.6,-0.35450854897499084,2885\n253.0,50.6,-0.33882418274879456,2890\n258.0,51.6,-0.3370402753353119,2895\n222.0,44.4,-0.3146362602710724,2900\n356.0,71.2,-0.3271253705024719,2905\n280.0,56.0,-0.3200274407863617,2910\n359.0,71.8,-0.3188944458961487,2915\n379.0,75.8,-0.3070104420185089,2920\n340.0,68.0,-0.3329097330570221,2925\n234.0,46.8,-0.3134278953075409,2930\n399.0,79.8,-0.3333054184913635,2935\n382.0,76.4,-0.32842057943344116,2940\n338.0,67.6,-0.32705143094062805,2945\n396.0,79.2,-0.31444960832595825,2950\n294.0,58.8,-0.34350016713142395,2955\n279.0,55.8,-0.3429255187511444,2960\n297.0,59.4,-0.3333609700202942,2965\n312.0,62.4,-0.3460474908351898,2970\n356.0,71.2,-0.36129605770111084,2975\n360.0,72.0,-0.33534759283065796,2980\n417.0,83.4,-0.31144455075263977,2985\n307.0,61.4,-0.3263359069824219,2990\n446.0,89.2,-0.33677947521209717,2995\n308.0,61.6,-0.31382012367248535,3000\n409.0,81.8,-0.3109150230884552,3005\n313.0,62.6,-0.33344486355781555,3010\n360.0,72.0,-0.3305971026420593,3015\n380.0,76.0,-0.33848661184310913,3020\n428.0,85.6,-0.33520564436912537,3025\n417.0,83.4,-0.3054162859916687,3030\n361.0,72.2,-0.3295208215713501,3035\n339.0,67.8,-0.32029515504837036,3040\n324.0,64.8,-0.3317856788635254,3045\n345.0,69.0,-0.32017967104911804,3050\n355.0,71.0,-0.35359203815460205,3055\n446.0,89.2,-0.31876492500305176,3060\n512.0,102.4,-0.3052927553653717,3065\n287.0,57.4,-0.30824708938598633,3070\n393.0,78.6,-0.30933746695518494,3075\n488.0,97.6,-0.2817961871623993,3080\n546.0,109.2,-0.3001251816749573,3085\n360.0,72.0,-0.324937105178833,3090\n339.0,67.8,-0.31537944078445435,3095\n426.0,85.2,-0.30676764249801636,3100\n359.0,71.8,-0.32886287569999695,3105\n324.0,64.8,-0.3065482974052429,3110\n291.0,58.2,-0.3147521913051605,3115\n409.0,81.8,-0.301559716463089,3120\n384.0,76.8,-0.32866623997688293,3125\n496.0,99.2,-0.31302568316459656,3130\n354.0,70.8,-0.3215748965740204,3135\n331.0,66.2,-0.3109250068664551,3140\n492.0,98.4,-0.30740219354629517,3145\n357.0,71.4,-0.30839717388153076,3150\n486.0,97.2,-0.3035200834274292,3155\n349.0,69.8,-0.34934717416763306,3160\n351.0,70.2,-0.32749202847480774,3165\n249.0,49.8,-0.33485740423202515,3170\n260.0,52.0,-0.31605178117752075,3175\n474.0,94.8,-0.29330599308013916,3180\n307.0,61.4,-0.323680579662323,3185\n247.0,49.4,-0.3127794861793518,3190\n237.0,47.4,-0.32012975215911865,3195\n209.0,41.8,-0.3009372353553772,3200\n264.0,52.8,-0.31981420516967773,3205\n266.0,53.2,-0.3103926479816437,3210\n204.0,40.8,-0.3112174868583679,3215\n300.0,60.0,-0.3256223201751709,3220\n263.0,52.6,-0.32424187660217285,3225\n199.0,39.8,-0.32241830229759216,3230\n179.0,35.8,-0.29255419969558716,3235\n163.0,32.6,-0.29931506514549255,3240\n168.0,33.6,-0.28654876351356506,3245\n203.0,40.6,-0.2834084928035736,3250\n226.0,45.2,-0.3224877417087555,3255\n261.0,52.2,-0.28535372018814087,3260\n227.0,45.4,-0.316785991191864,3265\n127.0,25.4,-0.29044753313064575,3270\n199.0,39.8,-0.3178306818008423,3275\n179.0,35.8,-0.28865891695022583,3280\n210.0,42.0,-0.31542903184890747,3285\n223.0,44.6,-0.3022362291812897,3290\n298.0,59.6,-0.3037247657775879,3295\n265.0,53.0,-0.30964574217796326,3300\n264.0,52.8,-0.30173414945602417,3305\n256.0,51.2,-0.3340996503829956,3310\n313.0,62.6,-0.33198970556259155,3315\n357.0,71.4,-0.31368961930274963,3320\n388.0,77.6,-0.3070986270904541,3325\n369.0,73.8,-0.32399335503578186,3330\n292.0,58.4,-0.3216218650341034,3335\n304.0,60.8,-0.3278127610683441,3340\n376.0,75.2,-0.3330898880958557,3345\n502.0,100.4,-0.30079925060272217,3350\n446.0,89.2,-0.3284432590007782,3355\n325.0,65.0,-0.32955601811408997,3360\n226.0,45.2,-0.3306828439235687,3365\n553.0,110.6,-0.29716169834136963,3370\n441.0,88.2,-0.30524060130119324,3375\n405.0,81.0,-0.3227323591709137,3380\n287.0,57.4,-0.3213828504085541,3385\n561.0,112.2,-0.30946630239486694,3390\n495.0,99.0,-0.29987454414367676,3395\n322.0,64.4,-0.3116146922111511,3400\n375.0,75.0,-0.3172515332698822,3405\n335.0,67.0,-0.31124642491340637,3410\n347.0,69.4,-0.3313350975513458,3415\n447.0,89.4,-0.3128863573074341,3420\n443.0,88.6,-0.323086678981781,3425\n372.0,74.4,-0.33466094732284546,3430\n390.0,78.0,-0.32339927554130554,3435\n336.0,67.2,-0.341411828994751,3440\n371.0,74.2,-0.31383463740348816,3445\n224.0,44.8,-0.3281266987323761,3450\n512.0,102.4,-0.3126053512096405,3455\n410.0,82.0,-0.28869909048080444,3460\n401.0,80.2,-0.3258347809314728,3465\n398.0,79.6,-0.30318811535835266,3470\n367.0,73.4,-0.3302140533924103,3475\n470.0,94.0,-0.29585710167884827,3480\n406.0,81.2,-0.3037850260734558,3485\n333.0,66.6,-0.309684693813324,3490\n515.0,103.0,-0.3057820796966553,3495\n511.0,102.2,-0.2935052514076233,3500\n265.0,53.0,-0.3512534499168396,3505\n313.0,62.6,-0.30854520201683044,3510\n589.0,117.8,-0.2876599431037903,3515\n418.0,83.6,-0.30609408020973206,3520\n337.0,67.4,-0.33163246512413025,3525\n352.0,70.4,-0.28263986110687256,3530\n225.0,45.0,-0.32982054352760315,3535\n443.0,88.6,-0.3075873553752899,3540\n321.0,64.2,-0.312394380569458,3545\n319.0,63.8,-0.30340978503227234,3550\n559.0,111.8,-0.29932039976119995,3555\n367.0,73.4,-0.32565683126449585,3560\n340.0,68.0,-0.3189055323600769,3565\n383.0,76.6,-0.3060702681541443,3570\n524.0,104.8,-0.2893108129501343,3575\n422.0,84.4,-0.31200963258743286,3580\n406.0,81.2,-0.3110651969909668,3585\n580.0,116.0,-0.3003738820552826,3590\n521.0,104.2,-0.28827184438705444,3595\n594.0,118.8,-0.2962532043457031,3600\n532.0,106.4,-0.2853490114212036,3605\n310.0,62.0,-0.32882094383239746,3610\n571.0,114.2,-0.29556986689567566,3615\n391.0,78.2,-0.30969303846359253,3620\n358.0,71.6,-0.35239845514297485,3625\n522.0,104.4,-0.2969819903373718,3630\n469.0,93.8,-0.278849333524704,3635\n408.0,81.6,-0.29693683981895447,3640\n615.0,123.0,-0.27214157581329346,3645\n516.0,103.2,-0.2786034643650055,3650\n476.0,95.2,-0.28871846199035645,3655\n524.0,104.8,-0.2695292532444,3660\n604.0,120.8,-0.2654711604118347,3665\n579.0,115.8,-0.2629389762878418,3670\n640.0,128.0,-0.24999983608722687,3675\n633.0,126.6,-0.2714177370071411,3680\n513.0,102.6,-0.288544237613678,3685\n518.0,103.6,-0.2821209132671356,3690\n408.0,81.6,-0.2815868556499481,3695\n472.0,94.4,-0.29285475611686707,3700\n433.0,86.6,-0.28891414403915405,3705\n448.0,89.6,-0.283041775226593,3710\n417.0,83.4,-0.2867187559604645,3715\n427.0,85.4,-0.29195308685302734,3720\n453.0,90.6,-0.30142688751220703,3725\n456.0,91.2,-0.30443498492240906,3730\n443.0,88.6,-0.29843389987945557,3735\n491.0,98.2,-0.2915704548358917,3740\n444.0,88.8,-0.28764721751213074,3745\n406.0,81.2,-0.29444530606269836,3750\n564.0,112.8,-0.24998706579208374,3755\n535.0,107.0,-0.28250670433044434,3760\n426.0,85.2,-0.2872706353664398,3765\n451.0,90.2,-0.26859310269355774,3770\n559.0,111.8,-0.26078301668167114,3775\n483.0,96.6,-0.2515645921230316,3780\n500.0,100.0,-0.24650022387504578,3785\n585.0,117.0,-0.24152213335037231,3790\n463.0,92.6,-0.26292717456817627,3795\n483.0,96.6,-0.2518559694290161,3800\n499.0,99.8,-0.2348634898662567,3805\n550.0,110.0,-0.26576220989227295,3810\n700.0,140.0,-0.240694060921669,3815\n560.0,112.0,-0.23301252722740173,3820\n579.0,115.8,-0.25920626521110535,3825\n560.0,112.0,-0.2541171908378601,3830\n598.0,119.6,-0.24800601601600647,3835\n401.0,80.2,-0.2769554555416107,3840\n933.0,186.6,-0.2157062292098999,3845\n666.0,133.2,-0.2362421602010727,3850\n664.0,132.8,-0.2353663593530655,3855\n459.0,91.8,-0.28326261043548584,3860\n590.0,118.0,-0.2582065761089325,3865\n770.0,154.0,-0.23853853344917297,3870\n741.0,148.2,-0.24289435148239136,3875\n801.0,160.2,-0.24660153687000275,3880\n885.0,177.0,-0.2432679533958435,3885\n995.0,199.0,-0.22121961414813995,3890\n893.0,178.6,-0.22451677918434143,3895\n1000.0,200.0,-0.21893183887004852,3900\n866.0,173.2,-0.2338653802871704,3905\n877.0,175.4,-0.23866155743598938,3910\n866.0,173.2,-0.22646315395832062,3915\n820.0,164.0,-0.23564893007278442,3920\n906.0,181.2,-0.2351033091545105,3925\n1000.0,200.0,-0.23822283744812012,3930\n1000.0,200.0,-0.2256428599357605,3935\n848.0,169.6,-0.22264470160007477,3940\n1000.0,200.0,-0.2249828279018402,3945\n850.0,170.0,-0.2305542677640915,3950\n832.0,166.4,-0.23433412611484528,3955\n877.0,175.4,-0.2427292764186859,3960\n862.0,172.4,-0.22772324085235596,3965\n933.0,186.6,-0.23778697848320007,3970\n949.0,189.8,-0.21746407449245453,3975\n889.0,177.8,-0.22645731270313263,3980\n955.0,191.0,-0.2358730137348175,3985\n937.0,187.4,-0.22857512533664703,3990\n882.0,176.4,-0.2432265728712082,3995\n892.0,178.4,-0.2350594699382782,4000\n758.0,151.6,-0.23316599428653717,4005\n922.0,184.4,-0.22859112918376923,4010\n776.0,155.2,-0.23555603623390198,4015\n744.0,148.8,-0.23500294983386993,4020\n689.0,137.8,-0.25206661224365234,4025\n723.0,144.6,-0.2562565505504608,4030\n597.0,119.4,-0.2625977396965027,4035\n833.0,166.6,-0.23476195335388184,4040\n686.0,137.2,-0.25012633204460144,4045\n693.0,138.6,-0.2593766748905182,4050\n626.0,125.2,-0.25423967838287354,4055\n699.0,139.8,-0.2497188150882721,4060\n629.0,125.8,-0.23464636504650116,4065\n545.0,109.0,-0.22980281710624695,4070\n612.0,122.4,-0.2374291568994522,4075\n483.0,96.6,-0.24381035566329956,4080\n597.0,119.4,-0.24583974480628967,4085\n866.0,173.2,-0.21999792754650116,4090\n566.0,113.2,-0.236464262008667,4095\n520.0,104.0,-0.23599712550640106,4100\n710.0,142.0,-0.2367580533027649,4105\n636.0,127.2,-0.24715624749660492,4110\n651.0,130.2,-0.23304535448551178,4115\n890.0,178.0,-0.24069303274154663,4120\n627.0,125.4,-0.23890958726406097,4125\n911.0,182.2,-0.23527511954307556,4130\n987.0,197.4,-0.22301892936229706,4135\n964.0,192.8,-0.2309679090976715,4140\n906.0,181.2,-0.2259397804737091,4145\n885.0,177.0,-0.23679006099700928,4150\n1000.0,200.0,-0.21609149873256683,4155\n971.0,194.2,-0.22595475614070892,4160\n957.0,191.4,-0.24088460206985474,4165\n745.0,149.0,-0.236735999584198,4170\n830.0,166.0,-0.23554329574108124,4175\n969.0,193.8,-0.24249093234539032,4180\n940.0,188.0,-0.2346430867910385,4185\n960.0,192.0,-0.2440900355577469,4190\n921.0,184.2,-0.2521948516368866,4195\n937.0,187.4,-0.23159390687942505,4200\n895.0,179.0,-0.25293585658073425,4205\n871.0,174.2,-0.2548487186431885,4210\n985.0,197.0,-0.2381192296743393,4215\n856.0,171.2,-0.2526217997074127,4220\n886.0,177.2,-0.24826432764530182,4225\n915.0,183.0,-0.2533659338951111,4230\n955.0,191.0,-0.232761412858963,4235\n1000.0,200.0,-0.2348642200231552,4240\n1000.0,200.0,-0.23721769452095032,4245\n905.0,181.0,-0.24588888883590698,4250\n999.0,199.8,-0.23869076371192932,4255\n928.0,185.6,-0.2406618446111679,4260\n869.0,173.8,-0.26329174637794495,4265\n916.0,183.2,-0.24832451343536377,4270\n849.0,169.8,-0.26125943660736084,4275\n884.0,176.8,-0.25208091735839844,4280\n674.0,134.8,-0.27581986784935,4285\n875.0,175.0,-0.2634401023387909,4290\n817.0,163.4,-0.2539278566837311,4295\n762.0,152.4,-0.26039013266563416,4300\n818.0,163.6,-0.25604119896888733,4305\n947.0,189.4,-0.24396267533302307,4310\n704.0,140.8,-0.2625954747200012,4315\n835.0,167.0,-0.2363101691007614,4320\n904.0,180.8,-0.2511977255344391,4325\n1000.0,200.0,-0.2310427725315094,4330\n885.0,177.0,-0.24242180585861206,4335\n852.0,170.4,-0.24381664395332336,4340\n744.0,148.8,-0.2467358559370041,4345\n1000.0,200.0,-0.22906582057476044,4350\n905.0,181.0,-0.2419399917125702,4355\n1000.0,200.0,-0.21317607164382935,4360\n1000.0,200.0,-0.22479897737503052,4365\n925.0,185.0,-0.23383106291294098,4370\n999.0,199.8,-0.21604128181934357,4375\n977.0,195.4,-0.22164760529994965,4380\n957.0,191.4,-0.20744213461875916,4385\n877.0,175.4,-0.23589551448822021,4390\n1000.0,200.0,-0.21450161933898926,4395\n984.0,196.8,-0.21658353507518768,4400\n943.0,188.6,-0.2249835729598999,4405\n964.0,192.8,-0.2135917693376541,4410\n1000.0,200.0,-0.21524985134601593,4415\n999.0,199.8,-0.20534180104732513,4420\n957.0,191.4,-0.2311737984418869,4425\n931.0,186.2,-0.23489539325237274,4430\n952.0,190.4,-0.22387759387493134,4435\n1000.0,200.0,-0.20473726093769073,4440\n1000.0,200.0,-0.20156295597553253,4445\n957.0,191.4,-0.21567095816135406,4450\n1000.0,200.0,-0.21938931941986084,4455\n1000.0,200.0,-0.20336979627609253,4460\n1000.0,200.0,-0.20344780385494232,4465\n999.0,199.8,-0.21003496646881104,4470\n1000.0,200.0,-0.21750986576080322,4475\n968.0,193.6,-0.22262828052043915,4480\n1000.0,200.0,-0.21291951835155487,4485\n1000.0,200.0,-0.21631832420825958,4490\n1000.0,200.0,-0.21128293871879578,4495\n951.0,190.2,-0.22381260991096497,4500\n1000.0,200.0,-0.2149495780467987,4505\n1000.0,200.0,-0.1978849470615387,4510\n1000.0,200.0,-0.1983804553747177,4515\n1000.0,200.0,-0.21148160099983215,4520\n1000.0,200.0,-0.22023101150989532,4525\n1000.0,200.0,-0.2018955647945404,4530\n1000.0,200.0,-0.21775908768177032,4535\n1000.0,200.0,-0.21060989797115326,4540\n1000.0,200.0,-0.22002004086971283,4545\n951.0,190.2,-0.22761140763759613,4550\n1000.0,200.0,-0.20279309153556824,4555\n992.0,198.4,-0.21495231986045837,4560\n849.0,169.8,-0.24093472957611084,4565\n1000.0,200.0,-0.21322326362133026,4570\n978.0,195.6,-0.22428254783153534,4575\n1000.0,200.0,-0.21417734026908875,4580\n1000.0,200.0,-0.20100240409374237,4585\n1000.0,200.0,-0.20599228143692017,4590\n1000.0,200.0,-0.23221741616725922,4595\n940.0,188.0,-0.22649583220481873,4600\n974.0,194.8,-0.22526900470256805,4605\n940.0,188.0,-0.23177145421504974,4610\n1000.0,200.0,-0.2212996780872345,4615\n1000.0,200.0,-0.22226651012897491,4620\n1000.0,200.0,-0.20896723866462708,4625\n1000.0,200.0,-0.23121048510074615,4630\n1000.0,200.0,-0.222879558801651,4635\n1000.0,200.0,-0.2117454558610916,4640\n1000.0,200.0,-0.23424331843852997,4645\n1000.0,200.0,-0.23412813246250153,4650\n971.0,194.2,-0.2223711609840393,4655\n1000.0,200.0,-0.22414419054985046,4660\n1000.0,200.0,-0.2104741632938385,4665\n1000.0,200.0,-0.23048822581768036,4670\n1000.0,200.0,-0.2183581441640854,4675\n1000.0,200.0,-0.23302426934242249,4680\n1000.0,200.0,-0.22289453446865082,4685\n1000.0,200.0,-0.21986311674118042,4690\n996.0,199.2,-0.23436616361141205,4695\n1000.0,200.0,-0.2223847359418869,4700\n1000.0,200.0,-0.22365456819534302,4705\n1000.0,200.0,-0.23042644560337067,4710\n1000.0,200.0,-0.22056877613067627,4715\n1000.0,200.0,-0.22903500497341156,4720\n797.0,159.4,-0.2448205053806305,4725\n984.0,196.8,-0.23969906568527222,4730\n918.0,183.6,-0.24919341504573822,4735\n1000.0,200.0,-0.24023643136024475,4740\n979.0,195.8,-0.24294376373291016,4745\n905.0,181.0,-0.2351226955652237,4750\n993.0,198.6,-0.21702656149864197,4755\n1000.0,200.0,-0.22933296859264374,4760\n999.0,199.8,-0.23117783665657043,4765\n1000.0,200.0,-0.21749353408813477,4770\n981.0,196.2,-0.23020605742931366,4775\n1000.0,200.0,-0.22406738996505737,4780\n1000.0,200.0,-0.20574606955051422,4785\n955.0,191.0,-0.23178116977214813,4790\n1000.0,200.0,-0.20908139646053314,4795\n1000.0,200.0,-0.2139613926410675,4800\n1000.0,200.0,-0.22396966814994812,4805\n1000.0,200.0,-0.210700124502182,4810\n1000.0,200.0,-0.22813169658184052,4815\n1000.0,200.0,-0.21640326082706451,4820\n1000.0,200.0,-0.23403076827526093,4825\n1000.0,200.0,-0.2196320742368698,4830\n1000.0,200.0,-0.22289463877677917,4835\n1000.0,200.0,-0.23845568299293518,4840\n1000.0,200.0,-0.21450844407081604,4845\n1000.0,200.0,-0.22363585233688354,4850\n1000.0,200.0,-0.20612011849880219,4855\n1000.0,200.0,-0.20918257534503937,4860\n1000.0,200.0,-0.23008960485458374,4865\n1000.0,200.0,-0.2219572514295578,4870\n910.0,182.0,-0.2174006849527359,4875\n1000.0,200.0,-0.21942415833473206,4880\n1000.0,200.0,-0.2067996859550476,4885\n1000.0,200.0,-0.21193228662014008,4890\n1000.0,200.0,-0.2176903337240219,4895\n1000.0,200.0,-0.20393991470336914,4900\n1000.0,200.0,-0.21651402115821838,4905\n966.0,193.2,-0.216569185256958,4910\n1000.0,200.0,-0.22076433897018433,4915\n979.0,195.8,-0.2166718691587448,4920\n1000.0,200.0,-0.2253253012895584,4925\n1000.0,200.0,-0.239028200507164,4930\n1000.0,200.0,-0.2164914757013321,4935\n1000.0,200.0,-0.20864802598953247,4940\n1000.0,200.0,-0.2230301946401596,4945\n980.0,196.0,-0.23694653809070587,4950\n907.0,181.4,-0.23473422229290009,4955\n973.0,194.6,-0.22844122350215912,4960\n1000.0,200.0,-0.22364303469657898,4965\n881.0,176.2,-0.23789668083190918,4970\n1000.0,200.0,-0.21272939443588257,4975\n974.0,194.8,-0.2289992868900299,4980\n862.0,172.4,-0.23540543019771576,4985\n914.0,182.8,-0.23260311782360077,4990\n737.0,147.4,-0.2543960213661194,4995\n" }, { "path": "history/ppo1.csv", "content": "Episode_reward,episode\n-1037.2442064560664,0\n-1847.6011742012683,1\n-1572.0879549063445,2\n-1909.3375551414695,3\n-1625.804678424486,4\n-1458.3791532216028,5\n-1676.7294710872982,6\n-1880.5057794936977,7\n-1714.7693216217642,8\n-1823.5511048861317,9\n-1646.443571113762,10\n-1791.0004302144125,11\n-1665.68509294836,12\n-1539.889248662342,13\n-1767.7738309574574,14\n-1572.8167680134982,15\n-1782.487852652908,16\n-1627.7773814954378,17\n-1798.02694320105,18\n-1043.1123802930633,19\n-1824.8765786915278,20\n-1738.5748540276559,21\n-1615.4604821214007,22\n-1615.866218962353,23\n-1756.9861376765596,24\n-1776.4853929124988,25\n-1571.456568416333,26\n-1731.0225055701799,27\n-1587.0271869127716,28\n-1617.4324253980737,29\n-1675.8339629364875,30\n-1589.88151645331,31\n-1520.736910439898,32\n-1703.4725034879411,33\n-1592.095144261398,34\n-1488.159085305198,35\n-1655.3300268675887,36\n-1416.3679355346933,37\n-1433.1090706415507,38\n-1531.0053852655142,39\n-1567.0351726159126,40\n-1588.564338627357,41\n-1182.073475448932,42\n-1475.0963413565785,43\n-1454.7558679603126,44\n-1229.3876682396108,45\n-1468.3415804526874,46\n-1343.0820826356976,47\n-1513.5150837470433,48\n-1060.864898279517,49\n-1466.2970468041235,50\n-1466.4791867410315,51\n-1414.8144066165612,52\n-1404.5800318908703,53\n-1361.4837234088805,54\n-1356.8567702395721,55\n-1348.8952982791732,56\n-1413.3690608311895,57\n-1375.0557294954053,58\n-1321.5190279442513,59\n-1372.1536567426783,60\n-1140.597696688407,61\n-1402.6687097798392,62\n-1574.6781229328142,63\n-1576.7077879607637,64\n-1561.1441634778998,65\n-1200.2874926818092,66\n-1222.3030216899904,67\n-1080.4847657134544,68\n-1439.779979755708,69\n-1406.331773423733,70\n-1052.8787347372734,71\n-1557.7074680390858,72\n-1559.689294638684,73\n-1561.5088918414517,74\n-1113.9055807595837,75\n-1315.7917468029939,76\n-1351.3446651057125,77\n-1454.1523051798201,78\n-1309.8238089783406,79\n-1268.8879586322907,80\n-1194.4745312273606,81\n-1389.2316932804683,82\n-1270.7457267870166,83\n-1333.9654163853515,84\n-1338.753110585217,85\n-1398.082088065302,86\n-1138.7920123870028,87\n-1346.608755810008,88\n-1281.9234448097484,89\n-1324.6354138259035,90\n-1236.892077361457,91\n-1472.552788664081,92\n-1217.9659027640732,93\n-1170.3323537084118,94\n-1179.7302522054513,95\n-1199.4833544268613,96\n-1097.9397919020873,97\n-1172.5576964875197,98\n-1524.7203645772004,99\n-1516.3794741179613,100\n-1171.0919390972815,101\n-1292.1748244736343,102\n-1374.3293422127795,103\n-1100.207684511091,104\n-1352.3333838116143,105\n-1090.0873226110825,106\n-1313.6740663009105,107\n-1327.38168174021,108\n-1083.1294573788716,109\n-983.6142335865583,110\n-881.3843361284513,111\n-777.6955586752294,112\n-1523.2942760021479,113\n-643.1245561336376,114\n-967.2842059822092,115\n-1363.9080690602211,116\n-1536.377511573882,117\n-1060.1031102279994,118\n-1127.3393896240152,119\n-1355.9448272702484,120\n-1383.99389366388,121\n-1508.803686267915,122\n-1349.5714176609602,123\n-1348.4093299734961,124\n-979.3129210110255,125\n-1109.0927160725516,126\n-1494.3358662370995,127\n-1232.529477955824,128\n-1198.2221484953939,129\n-1204.3422832071053,130\n-1223.259894601989,131\n-1048.264869873165,132\n-1214.9576570813924,133\n-1233.7189467355558,134\n-781.7075925291913,135\n-1353.194273574951,136\n-685.7806108704868,137\n-1287.1007558246267,138\n-1230.9323991845604,139\n-1186.4067360770057,140\n-786.4065882045364,141\n-1001.600194957268,142\n-915.6642388726257,143\n-1092.0082532491876,144\n-675.889350695875,145\n-873.1397630522355,146\n-1184.5470871826124,147\n-1041.735120338675,148\n-645.3836706510756,149\n-594.7457941205153,150\n-924.9620007988684,151\n-525.3225826681372,152\n-515.1276436894625,153\n-1383.728730884463,154\n-525.0847542187588,155\n-730.2522633554491,156\n-588.412968141994,157\n-644.2747986192634,158\n-965.5322075506963,159\n-1413.4680491967438,160\n-1160.905794333069,161\n-653.7941212692359,162\n-784.4573284539567,163\n-1060.4051048850272,164\n-917.8574134621971,165\n-654.9807166763895,166\n-754.147898396826,167\n-521.2704617968814,168\n-1328.2391181587886,169\n-1229.9026965641324,170\n-520.406713854922,171\n-540.8686159112228,172\n-652.6905320159613,173\n-1083.3322644848085,174\n-518.2184614659982,175\n-520.5349799468748,176\n-643.2233794775584,177\n-1417.944931635821,178\n-390.4077956799553,179\n-652.8597513827308,180\n-1101.2712486358153,181\n-516.4330096473971,182\n-638.7717540256594,183\n-659.9938717646102,184\n-511.3881267579837,185\n-390.85780135927683,186\n-392.2780789936845,187\n-812.4970002494532,188\n-1415.7206853874468,189\n-783.2809505461622,190\n-434.011046630424,191\n-1140.6721625852579,192\n-911.2346112488109,193\n-1038.8824870828973,194\n-953.7735338040277,195\n-832.0679492311945,196\n-391.5803152777578,197\n-594.7326684632785,198\n-134.36126154017978,199\n-263.1388449951507,200\n-263.35297556848843,201\n-2.066724454856968,202\n-263.7467460787438,203\n-547.8040819121052,204\n-396.19772514454803,205\n-1.5163648546321538,206\n-1072.4688682420128,207\n-537.4239657853051,208\n-406.2957933943553,209\n-812.1427491763986,210\n-524.538393880087,211\n-130.09623397331515,212\n-546.0428094436602,213\n-1094.4604125441156,214\n-1049.990387896455,215\n-215.16348544943915,216\n-663.9809345973752,217\n-132.85756077729036,218\n-265.1615309177778,219\n-839.9940494275294,220\n-522.7645163607622,221\n-382.3034934689336,222\n-264.7068957473173,223\n-262.36920139541087,224\n-696.4631737266078,225\n-683.2932880126467,226\n-693.0030169321484,227\n-255.9343966393008,228\n-382.15861079855944,229\n-133.72960816436733,230\n-135.63361762344363,231\n-1070.3947800142726,232\n-261.77685016344583,233\n-786.6645231796923,234\n-524.2769697482614,235\n-904.6443397757624,236\n-395.87852836714035,237\n-395.7256764528719,238\n-130.2041242713439,239\n-258.95821903796764,240\n-392.74850252299376,241\n-932.9703316790616,242\n-130.53268719654994,243\n-133.11015787559413,244\n-262.90512124240774,245\n-397.52978082155977,246\n-806.9252845000568,247\n-404.94792012056337,248\n-1.344683358323401,249\n-265.03263166243835,250\n-393.69235487465875,251\n-397.18762513507096,252\n-261.93852467722917,253\n-0.6932713815189028,254\n-0.2772104986206239,255\n-134.84329433614684,256\n-0.3018889248722081,257\n-262.4557578330868,258\n-410.7676257379418,259\n-0.20614191128882048,260\n-135.90980010963827,261\n-1.5306243617420219,262\n-0.2775277070912824,263\n-262.20553184217437,264\n-262.75668594102484,265\n-265.478691889229,266\n-649.5840602275849,267\n-2.3359025016926287,268\n-134.37728184841674,269\n-403.79532548294,270\n-131.44685457390344,271\n-2.540079220521762,272\n-0.6245710844806219,273\n-552.3203313210981,274\n-420.7688029635937,275\n-673.1561228210983,276\n-566.6329676342501,277\n-826.7218302286809,278\n-133.19607412292007,279\n-261.41161243420396,280\n-130.1503432567707,281\n-745.9951207605527,282\n-593.1009909805875,283\n-422.63194819788635,284\n-264.352460975422,285\n-132.6281002698094,286\n-272.5944476952579,287\n-276.52496884585025,288\n-2.8595270844298355,289\n-134.3542481081264,290\n-863.0600084193688,291\n-0.5838006191185878,292\n-260.12384955626436,293\n-1092.3520075256804,294\n-134.47579544793888,295\n-263.08226456567047,296\n-263.14208863751736,297\n-403.6384862962421,298\n-676.3129769613263,299\n-259.5850824851148,300\n-0.5599792911842503,301\n-841.3971214232955,302\n-400.68013796627827,303\n-261.9411324355016,304\n-134.2541162358826,305\n-400.7488164167632,306\n-933.7718406562569,307\n-395.58837818645895,308\n-781.4991873840422,309\n-542.4794573052475,310\n-261.768990759702,311\n-411.4022209681378,312\n-908.6098705476467,313\n-572.054132096904,314\n-406.3605476140625,315\n-0.9560144286465458,316\n-419.8098582593684,317\n-135.8733465246141,318\n-677.8877177512418,319\n-1.061000351544841,320\n-420.8935070674562,321\n-410.50094553986,322\n-527.1414088279827,323\n-405.8131292952111,324\n-133.82488800602135,325\n-410.9355644212376,326\n-1138.5231045218584,327\n-265.60551146761236,328\n-550.6279081029494,329\n-541.8809285146793,330\n-262.00608491340074,331\n-138.26408244291625,332\n-539.0046046868695,333\n-263.20194280779947,334\n-558.728749138611,335\n-542.6315237626562,336\n-396.69494447112845,337\n-442.68983986420074,338\n-551.2123324409539,339\n-731.7927987021761,340\n-1009.4307379021903,341\n-130.3130437347376,342\n-703.3807523268074,343\n-829.610762955033,344\n-268.59947452925485,345\n-396.18740801744417,346\n-735.5741244350246,347\n-139.65813774324894,348\n-264.5108789752626,349\n-135.02709500526709,350\n-1076.030566209553,351\n-537.6501749525565,352\n-726.1607610041605,353\n-132.67641477529017,354\n-129.46648686544765,355\n-0.6651282195807512,356\n-833.3597025127152,357\n-398.07084913357073,358\n-536.8416493729624,359\n-276.5888057271474,360\n-129.91619176523687,361\n-552.5026032887117,362\n-132.61990212646703,363\n-1045.702934322954,364\n-136.72606810089414,365\n-1355.3987029443895,366\n-819.6105195876539,367\n-136.38162152875142,368\n-129.88447417757365,369\n-556.502164519348,370\n-867.8536695272958,371\n-3.1904222595638547,372\n-1214.2744927178414,373\n-399.04245115980603,374\n-267.0060835194837,375\n-597.6943316498787,376\n-400.03826166480053,377\n-136.40065631097923,378\n-403.3178441063344,379\n-129.41396951948283,380\n-600.7776614286838,381\n-414.1901618076007,382\n-546.7017304910571,383\n-267.75816212558755,384\n-612.9366155119945,385\n-1058.307874636493,386\n-133.0263198111637,387\n-398.33869436455507,388\n-626.9282848155667,389\n-1.456315515841496,390\n-1070.830576551045,391\n-2.915776200583376,392\n-263.8466150169842,393\n-796.4792493574953,394\n-265.7777444052896,395\n-3.8084763401136645,396\n-606.3397790085836,397\n-406.5259438648021,398\n-264.4538015852596,399\n-421.1294585179861,400\n-268.94297207424785,401\n-264.328347182195,402\n-425.5756864303751,403\n-407.931805660131,404\n-1030.610987231829,405\n-538.4649919067501,406\n-3.6747535767855886,407\n-678.8135259365788,408\n-686.1830668911863,409\n-273.6093020323261,410\n-132.41383735644447,411\n-264.83687975071825,412\n-818.0039043899321,413\n-609.9351631453789,414\n-128.80103209940646,415\n-268.2065650529604,416\n-135.85894383504206,417\n-271.5610220919052,418\n-693.9040568212417,419\n-1.8509163368128447,420\n-410.9801257663261,421\n-853.0493149225435,422\n-133.21021612525206,423\n-0.8437241243893877,424\n-130.65573438789744,425\n-542.845174654451,426\n-135.6855244155601,427\n-417.52309785513864,428\n-0.7581486423250411,429\n-266.26996934460556,430\n-263.94899093386437,431\n-736.3778654066004,432\n-133.66564996545347,433\n-267.49172424982123,434\n-133.87407892344285,435\n-132.2231734146608,436\n-555.2525216747292,437\n-1.8491548042185535,438\n-565.7249895439152,439\n-404.9796413234927,440\n-267.8873702005579,441\n-1.5675334840621846,442\n-129.49254983614827,443\n-1300.3818970915056,444\n-266.80829310210686,445\n-132.846667239015,446\n-410.35682063110113,447\n-760.7076317443073,448\n-1.0388158036790809,449\n-562.2060873809967,450\n-407.10572427607997,451\n-136.4642818575228,452\n-3.06135888344035,453\n-1221.50293375514,454\n-136.77496013669295,455\n-737.2495485798808,456\n-137.50006864952155,457\n-130.33342296721955,458\n-4.692447559268221,459\n-3.0063944854129865,460\n-135.00453947709414,461\n-397.64065865936266,462\n-3.561275635032984,463\n-780.3775885951651,464\n-532.2664055760271,465\n-131.92450427568124,466\n-1.6046697510118604,467\n-133.00177858870154,468\n-269.9109942657057,469\n-409.1694204055191,470\n-603.3617792436297,471\n-1.3767554201233199,472\n-413.17201205566533,473\n-532.8892182144033,474\n-263.2455839262573,475\n-277.02963483896184,476\n-560.1389932556149,477\n-571.890657804561,478\n-403.8028746209018,479\n-804.407380454556,480\n-1.0695254030414343,481\n-416.2029772547103,482\n-133.83507168589958,483\n-940.8086921348546,484\n-408.28469089165543,485\n-274.042746604528,486\n-589.1994466203586,487\n-788.5111259162453,488\n-567.6472927943756,489\n-2.8289954734150604,490\n-685.3729429161508,491\n-268.206649921637,492\n-437.69087101363084,493\n-417.7660125988834,494\n-265.45291320659754,495\n-133.5340688725064,496\n-131.92462695915873,497\n-135.6125883822286,498\n-131.48925605691042,499\n-132.54677627855565,500\n-1.3503200153108306,501\n-128.38851899166932,502\n-134.89278397253204,503\n-571.4941378304989,504\n-133.89852908237478,505\n-544.2495575714942,506\n-662.4776823050677,507\n-394.3751616321957,508\n-3.095723819440499,509\n-273.44992714321717,510\n-264.10554627139595,511\n-135.41272508779565,512\n-134.09461308881612,513\n-137.0507207687898,514\n-770.5338478671764,515\n-437.8700667846867,516\n-0.3753059704914458,517\n-267.24315427654955,518\n-130.90939395108637,519\n-934.8662767688785,520\n-2.6669074896824068,521\n-400.62364325290656,522\n-1.6696647518102259,523\n-265.32637769376527,524\n-3.1507638004265694,525\n-131.12331418116048,526\n-141.87079055886477,527\n-825.050796846192,528\n-130.93318451514702,529\n-523.2001052956139,530\n-265.2578156473554,531\n-138.71887535890193,532\n-569.5915371523467,533\n-410.3924023954314,534\n-434.12030888844373,535\n-413.48962556770095,536\n-130.76640633828526,537\n-5.477341425270173,538\n-271.2594014584768,539\n-2.3942053220953805,540\n-416.8977422409377,541\n-137.71946218438592,542\n-447.78386376286915,543\n-593.1452045212368,544\n-268.15562890671407,545\n-137.01916305869315,546\n-416.6005802248519,547\n-133.14733860717726,548\n-955.0844258834192,549\n-541.828003387883,550\n-129.90083961519875,551\n-564.823565856297,552\n-673.1695141333834,553\n-431.8817699163025,554\n-414.0402790834923,555\n-3.3655643636069996,556\n-267.5634643918511,557\n-426.51625798642897,558\n-263.8942914418401,559\n-137.16189062976198,560\n-132.93351156595426,561\n-571.9126026765111,562\n-134.16517216498582,563\n-135.92685840435846,564\n-476.30809690606503,565\n-0.8487337210298733,566\n-641.6132162259648,567\n-1068.876620157741,568\n-137.4258776962059,569\n-136.0769507146851,570\n-4.703060983922232,571\n-407.1706781572831,572\n-270.2963991120964,573\n-564.1669740975232,574\n-137.26561702823415,575\n-266.5489372184778,576\n-137.13210947421558,577\n-270.831982385586,578\n-1.7468141487159978,579\n-1.4849363480522857,580\n-129.3052778899183,581\n-418.84526061092686,582\n-134.82637082827796,583\n-397.94707127618955,584\n-798.7909598149064,585\n-265.7945752737649,586\n-135.70373214053893,587\n-0.853213719900561,588\n-660.824533280652,589\n-128.7023744805705,590\n-700.6344152658089,591\n-405.5282776016281,592\n-131.51661086950145,593\n-546.3483223814508,594\n-130.9552567930365,595\n-622.4203463856843,596\n-713.4782817193388,597\n-433.8173879566531,598\n-0.5954833428422462,599\n-134.18011521885535,600\n-391.86917168825954,601\n-131.34606823313518,602\n-429.72554615560506,603\n-263.93241789048795,604\n-411.18839199918614,605\n-136.23181759696217,606\n-135.42180418910223,607\n-566.5887427520848,608\n-269.40157445291453,609\n-0.44954082388355426,610\n-137.74951057291003,611\n-530.4969090969139,612\n-0.8408298660101461,613\n-655.0911054306512,614\n-270.9270011695244,615\n-735.1944281658946,616\n-445.1202478259294,617\n-131.31860950262353,618\n-537.1851532649281,619\n-398.3129171538527,620\n-612.1639018731215,621\n-0.9781154579751646,622\n-422.26262330769566,623\n-267.6002013641027,624\n-290.826611085219,625\n-463.112283701044,626\n-135.18495498482508,627\n-277.2061411384699,628\n-414.3811680777204,629\n-269.35973348923324,630\n-268.7147126557296,631\n-1031.8346843730008,632\n-135.48849750900612,633\n-265.5453035909327,634\n-835.7266924828297,635\n-399.1918198110442,636\n-403.6450851699637,637\n-547.2124249306119,638\n-684.8378414330883,639\n-1.599978927671113,640\n-1.6224929969650146,641\n-404.9876712525678,642\n-274.8703557293909,643\n-403.2988346780608,644\n-527.0666215847235,645\n-130.12849901799794,646\n-546.6154267814015,647\n-275.31917997110565,648\n-132.82000295235852,649\n-413.08071161699394,650\n-399.3589442704302,651\n-268.17525555030767,652\n-262.10552725322657,653\n-0.9507842448702704,654\n-402.7553695776257,655\n-135.5222338783557,656\n-284.988499816711,657\n-264.96813152647314,658\n-269.16726789211015,659\n-134.50071014782637,660\n-271.42301612005053,661\n-131.53811305766246,662\n-267.3829856176393,663\n-270.6087845082228,664\n-266.5785389016133,665\n-0.6770756546128163,666\n-128.07267641433373,667\n-137.09983433230872,668\n-417.6622887039994,669\n-407.2922440721843,670\n-3.1545969883794145,671\n-136.53340841309807,672\n-786.469150702965,673\n-536.1827925680751,674\n-133.51393928233892,675\n-2.9746378402259133,676\n-133.2474602224068,677\n-549.570724944704,678\n-134.8465363406278,679\n-562.3033937037092,680\n-134.5553921366925,681\n-132.07888978555943,682\n-128.90414064885408,683\n-132.23955405484227,684\n-267.97689421182037,685\n-513.217333916576,686\n-272.8588077826132,687\n-544.2886822100733,688\n-579.2113464917387,689\n-412.7848730650725,690\n-4.452065119071128,691\n-135.5373403599167,692\n-137.13816133226257,693\n-2.8009456855288954,694\n-277.4049066814325,695\n-262.8958185202623,696\n-267.79642589717946,697\n-135.13590425948547,698\n-134.93201836959054,699\n-412.5084776634811,700\n-404.98281126255813,701\n-397.46178347457567,702\n-263.25359893845115,703\n-401.6344148366646,704\n-270.7635799186573,705\n-1044.2792627940007,706\n-407.7268382940735,707\n-1047.78721497132,708\n-130.042122638779,709\n-2.427203810527852,710\n-277.8793903873562,711\n-268.492683261287,712\n-134.85260940573343,713\n-3.644970307291,714\n-3.3424485262225923,715\n-128.93301738564546,716\n-402.4013526791311,717\n-5.634604106978109,718\n-2.127549855304606,719\n-400.640710677243,720\n-409.19330803392506,721\n-131.71037639613488,722\n-132.35696839893305,723\n-129.869658978088,724\n-541.166617745306,725\n-265.93340519439687,726\n-412.9878742742597,727\n-143.97101975329414,728\n-267.98300931988047,729\n-590.5282060855008,730\n-132.70921753195773,731\n-416.95785032050094,732\n-271.33759715428033,733\n-422.0658013594705,734\n-454.6655669451039,735\n-135.68166517455742,736\n-265.88942799807523,737\n-133.35630057418973,738\n-644.3422690739726,739\n-129.07826894675364,740\n-132.8475803566894,741\n-1.2385169825975992,742\n-400.4980016295058,743\n-135.92472632024584,744\n-272.82560031170306,745\n-271.2080419129983,746\n-140.09632826622712,747\n-132.31732484571646,748\n-268.61662370008344,749\n-135.4928481585457,750\n-264.9822230039586,751\n-263.85216604242146,752\n-4.297654498622435,753\n-868.8347257257907,754\n-263.80539085519047,755\n-412.051847712486,756\n-688.9334061440629,757\n-131.41646975653762,758\n-264.74865994364126,759\n-265.47092869729886,760\n-456.81526706325735,761\n-546.9080282663508,762\n-1.973824107780018,763\n-269.21536540895534,764\n-1.713891911538705,765\n-579.5219816460741,766\n-2.160252417522419,767\n-131.52234722924243,768\n-266.79542376975394,769\n-134.80137751374332,770\n-437.98618295870943,771\n-131.31948275973093,772\n-276.6048567042357,773\n-417.1125412701325,774\n-402.0588963375871,775\n-131.77966085374746,776\n-1202.0240857562246,777\n-268.6454334180883,778\n-136.06573808456898,779\n-138.7392813264935,780\n-135.35185466118227,781\n-4.945771675882908,782\n-394.03132182864863,783\n-531.3405440124228,784\n-753.1354633439472,785\n-132.7387300401294,786\n-556.5602709311516,787\n-398.7284435502883,788\n-268.80703946798826,789\n-128.48511069897174,790\n-288.9078505132446,791\n-135.45369363399183,792\n-128.56961294680187,793\n-131.52469595199747,794\n-408.4862126062992,795\n-3.1726002982308428,796\n-426.3254546918405,797\n-129.68554813214348,798\n-259.2116831641247,799\n-129.38928788553392,800\n-0.7818924298953402,801\n-845.4938181824932,802\n-400.17491153479136,803\n-263.48759160474424,804\n-717.1131246557032,805\n-262.52775793067445,806\n-128.66857497966637,807\n-525.3706129540489,808\n-0.6076074427473337,809\n-552.8401608433018,810\n-557.1314891791872,811\n-415.7032671935188,812\n-134.2864471035512,813\n-665.0598428173886,814\n-1.813916585198168,815\n-274.40863296500163,816\n-264.506952040262,817\n-514.996684311894,818\n-562.6859613202619,819\n-441.858270384301,820\n-927.9393285148145,821\n-548.1257156079951,822\n-405.5235810167245,823\n-125.95985836550261,824\n-268.82549952038573,825\n-134.20540390787758,826\n-1.0840017773152988,827\n-279.5282060904265,828\n-264.4457808012585,829\n-133.93484903734085,830\n-532.7975043702237,831\n-630.9424205988677,832\n-131.93469518318892,833\n-668.7810549529356,834\n-396.3146116097206,835\n-131.70347877207968,836\n-554.1754739445275,837\n-279.4474425314443,838\n-573.6787205194121,839\n-747.1235497488071,840\n-772.4638342271185,841\n-655.7338174960156,842\n-1.207609595902526,843\n-460.47588553465323,844\n-262.0675194319991,845\n-2.9776672191231977,846\n-538.8670903974812,847\n-2.6978337282468923,848\n-559.7220631971884,849\n-538.6709472232267,850\n-563.6743328093427,851\n-1.842495384532607,852\n-544.8070854192966,853\n-2.285782164071175,854\n-398.71635996910504,855\n-400.6305946106565,856\n-556.7762188439027,857\n-266.251380830894,858\n-412.91546130698947,859\n-128.97746555922245,860\n-265.4313161617809,861\n-3.639142443695814,862\n-131.70412437450034,863\n-515.6242599346612,864\n-272.9866860081554,865\n-553.0954435643473,866\n-267.29957739081857,867\n-526.2860526594312,868\n-131.94190359513618,869\n-540.8808435517136,870\n-1.6178104461034766,871\n-268.10015494857873,872\n-402.7952078984712,873\n-1.9592077732384836,874\n-0.5216608559077789,875\n-402.1489835906614,876\n-553.4442031304726,877\n-269.9690046465971,878\n-276.1817119872923,879\n-271.1235599374055,880\n-275.94245267581005,881\n-582.7145935933603,882\n-261.3104823876538,883\n-263.83462714769854,884\n-760.2535455603813,885\n-0.8021753966521787,886\n-133.10322583122255,887\n-398.61746941298054,888\n-264.556898163982,889\n-661.1002272713685,890\n-537.450820584527,891\n-530.4395784258222,892\n-564.5278075109893,893\n-0.3944341016162952,894\n-389.5220573655856,895\n-1.0693089702676783,896\n-132.59160502596077,897\n-557.2586270572458,898\n-1.4186572673440174,899\n-580.4043250036906,900\n-0.27219965426706516,901\n-935.2286616630636,902\n-265.24537977974165,903\n-0.7553752427148439,904\n-557.1230570190648,905\n-584.1020167810515,906\n-262.9979977009651,907\n-409.9826707043166,908\n-551.3636160041265,909\n-553.7277744973321,910\n-267.353851028601,911\n-264.0271174044538,912\n-461.38711652235196,913\n-657.8928280673879,914\n-127.98534551871897,915\n-397.0817708981668,916\n-399.88595759763257,917\n-263.16526215384965,918\n-0.33294567732415464,919\n-266.7624206331044,920\n-262.4756259223824,921\n-402.6128277170327,922\n-418.37880172359155,923\n-268.488285734389,924\n-130.60819688926827,925\n-127.03966787673068,926\n-135.77831626564947,927\n-132.47986991728274,928\n-395.5454742846722,929\n-261.0110543918011,930\n-273.20536698545413,931\n-128.56525891072985,932\n-130.12424555240193,933\n-261.87684494872167,934\n-126.64205980284324,935\n-0.8250768876945536,936\n-136.83005990926765,937\n-394.34277569046054,938\n-137.84820071367795,939\n-518.591718133272,940\n-273.82584160776474,941\n-427.1610849871556,942\n-0.5591722775362037,943\n-0.5475435002920703,944\n-413.32632034572754,945\n-413.78364777463514,946\n-130.63792297023898,947\n-266.82256099442105,948\n-131.9751892054003,949\n-269.2975589269708,950\n-134.77823071473324,951\n-138.7903780877018,952\n-265.3171143777411,953\n-136.15535505148492,954\n-410.04956244185274,955\n-126.86073726657668,956\n-693.4942100270466,957\n-300.2755010341002,958\n-129.7554008974262,959\n-130.39142087614172,960\n-0.4281488868789242,961\n-1069.6403775779477,962\n-134.95521920726932,963\n-265.1961508640712,964\n-515.8982622390333,965\n-495.72717611815966,966\n-398.5976251797379,967\n-0.6020062184204914,968\n-402.1985302496086,969\n-402.71885038226674,970\n-391.577628383538,971\n-267.51785547574775,972\n-599.6098761088606,973\n-0.29378066803076747,974\n-125.64155893361814,975\n-138.56290329018307,976\n-265.71785438548307,977\n-552.5502379806186,978\n-543.19424899033,979\n-520.7314789046981,980\n-0.23690883768232124,981\n-574.3360710007381,982\n-436.7252197108877,983\n-396.32656322247,984\n-129.47329023308035,985\n-388.0145435092373,986\n-397.83494102476277,987\n-706.5536437099101,988\n-285.18821712709655,989\n-510.61704071486866,990\n-796.963078767408,991\n-279.8122481069468,992\n-267.7081978992145,993\n-129.611990412507,994\n-577.4946715585467,995\n-134.37423290496676,996\n-134.1569150625576,997\n-129.90626519604768,998\n-528.6909502627306,999\n" }, { "path": "history/ppo2.csv", "content": "Episode_reward,episode\n-1673.3837361587691,0\n-1611.2107172299563,1\n-1659.4160521895024,2\n-1617.9072409202724,3\n-1861.9679785277126,4\n-1763.910137087496,5\n-1658.4920855201797,6\n-1754.9157618553288,7\n-1771.638571076579,8\n-1635.0500733190356,9\n-1863.882293772463,10\n-1664.9519363991008,11\n-1836.3254177350832,12\n-1775.645768017405,13\n-1865.7059554598081,14\n-1920.414217966775,15\n-1884.8910375338226,16\n-1743.024879306623,17\n-1653.244234369963,18\n-1751.1522173890673,19\n-1881.8639983374635,20\n-1510.054349955367,21\n-1887.5303639858053,22\n-1650.4917305014794,23\n-1905.0681187020784,24\n-1875.819462571147,25\n-1820.3064531151408,26\n-1723.8747104845838,27\n-1602.2116608304307,28\n-1855.345759622926,29\n-1710.032672688522,30\n-1769.424270335167,31\n-1707.9905089081487,32\n-1641.477811993138,33\n-1518.7050062000985,34\n-1566.549386520303,35\n-1668.2834998549986,36\n-1563.8955009358124,37\n-1362.205846609126,38\n-1283.5080213907506,39\n-1354.54501676445,40\n-1472.7425773045209,41\n-1471.2995495830785,42\n-1375.8887630767326,43\n-1339.077764792099,44\n-1434.533515049349,45\n-1485.0339921528862,46\n-1341.0398820936869,47\n-1486.84721756297,48\n-1352.957296788103,49\n-1342.5273363530544,50\n-1340.0682630180129,51\n-1036.2075945454487,52\n-1276.6274271479326,53\n-1378.7286652280159,54\n-1383.1333938843554,55\n-1064.402679493027,56\n-1280.5296942855787,57\n-1270.1603060839977,58\n-1310.1768800998607,59\n-1195.7544995374515,60\n-1201.4174510889954,61\n-1242.3759303993406,62\n-1289.033183004484,63\n-1353.406651924718,64\n-1172.2720349219376,65\n-1307.4848178663754,66\n-1336.8995309970574,67\n-1302.5196522410472,68\n-896.4895524174703,69\n-1021.6573199932254,70\n-1220.7766933793941,71\n-1317.625342544375,72\n-1203.2769689452646,73\n-1036.1990807027873,74\n-1166.6188368012795,75\n-1081.1619119858392,76\n-864.5837941042004,77\n-1098.249813354747,78\n-995.9852197347332,79\n-1177.9396746958691,80\n-1080.2470194508992,81\n-1039.092759952932,82\n-1109.2317723717972,83\n-932.7793296730388,84\n-990.4120615287829,85\n-1058.5168296232578,86\n-942.1667228712881,87\n-879.7663118141093,88\n-752.0422928604002,89\n-888.3943144061936,90\n-1128.1769755407613,91\n-1034.5925816665876,92\n-846.7631204763816,93\n-969.0698894458254,94\n-1164.060822006457,95\n-903.8996592979744,96\n-1184.424706774562,97\n-682.046077453111,98\n-782.7897181424028,99\n-647.8582736454379,100\n-894.6590602212112,101\n-724.0908468127682,102\n-635.216429500116,103\n-638.1912089809953,104\n-898.6114814646173,105\n-1188.6158496113953,106\n-557.2692091355098,107\n-996.6846572786549,108\n-1073.3532881944932,109\n-649.7672728167229,110\n-1008.5572013439719,111\n-1048.5657625792385,112\n-1125.0998674557268,113\n-896.1498540600301,114\n-782.9270386638215,115\n-1121.313137741631,116\n-916.552424255541,117\n-763.5646666093456,118\n-393.4227657720568,119\n-928.9484237373965,120\n-923.6335547465371,121\n-801.2321028734696,122\n-373.2352059863441,123\n-917.6934151483332,124\n-843.6923046474043,125\n-1004.2469703913843,126\n-751.4240173938097,127\n-592.3909700170991,128\n-524.5322196630357,129\n-391.90170261493563,130\n-528.4153900660657,131\n-522.4180378005351,132\n-263.4760752753853,133\n-791.830124696192,134\n-674.5605915786391,135\n-857.1385714780329,136\n-1106.5356286265423,137\n-866.2994864080176,138\n-1033.2325725468008,139\n-1113.64326634444,140\n-1070.645549272555,141\n-391.23800823424483,142\n-385.3694897852698,143\n-248.7725308273716,144\n-253.69001182733678,145\n-707.954918440374,146\n-832.0992924120936,147\n-922.3908834435002,148\n-389.75089834507963,149\n-1005.8163725249925,150\n-667.399030809139,151\n-392.38536379812695,152\n-393.2914965567365,153\n-942.9196171950974,154\n-261.3504913784919,155\n-917.101682714707,156\n-128.8262003110178,157\n-780.9354786045765,158\n-521.7810461550926,159\n-426.7738867755489,160\n-265.61651436477365,161\n-819.7298783292246,162\n-272.9215813659676,163\n-776.834332431869,164\n-262.3900661545925,165\n-132.12623622513212,166\n-262.11746326066526,167\n-263.20980945053094,168\n-0.3805095273600947,169\n-1076.2878428885865,170\n-254.14938138935815,171\n-261.01856919115255,172\n-930.2858176349101,173\n-537.7008658919203,174\n-522.7103301860867,175\n-3.6120355148026184,176\n-130.21854631752979,177\n-434.84039322578195,178\n-2.230960383757409,179\n-1027.4463179253348,180\n-129.47132238408255,181\n-0.37866273150030755,182\n-385.27267868093935,183\n-259.89752785941494,184\n-0.7283559321068548,185\n-789.361067525375,186\n-126.28503326037485,187\n-539.3883895851371,188\n-256.48478145056396,189\n-2.2659392371830966,190\n-524.3759178071607,191\n-271.97892582579505,192\n-926.719571460742,193\n-129.15075877570163,194\n-548.5670445269479,195\n-393.8543897788036,196\n-258.06529336789964,197\n-527.2038407590632,198\n-2.371095343424757,199\n-582.3641558329555,200\n-403.3927851237764,201\n-130.0053149485939,202\n-524.4675766555405,203\n-1067.338794259848,204\n-129.4322522266625,205\n-387.9664545818074,206\n-465.4821728518156,207\n-254.82783040484972,208\n-130.94755085634196,209\n-1186.100938903983,210\n-2.35943540657512,211\n-523.6375459505396,212\n-522.4807096870338,213\n-133.20918183100372,214\n-401.03224261826705,215\n-402.69143238316764,216\n-126.45996737306298,217\n-659.5856896037266,218\n-404.1698843168999,219\n-130.07772170996324,220\n-130.59138001775293,221\n-1080.0004514668324,222\n-266.65057056851094,223\n-1114.9212461773673,224\n-131.55964489194,225\n-130.02418188328951,226\n-533.5183804943565,227\n-126.94992274084004,228\n-394.44267547163906,229\n-259.194533631094,230\n-125.92078014408864,231\n-128.39799643802465,232\n-0.44082835340058646,233\n-267.0063816747732,234\n-555.3847179395383,235\n-253.08091862138875,236\n-274.3863333818009,237\n-266.3580583235881,238\n-281.1009014282488,239\n-401.4191804534081,240\n-262.1729832136079,241\n-790.4367453057972,242\n-0.9034114207850806,243\n-532.5229837613709,244\n-660.4134402241634,245\n-423.7315171563312,246\n-1.228469885314944,247\n-131.68273697298633,248\n-131.2780220185061,249\n-396.03978805841376,250\n-0.6360345163838601,251\n-128.73650255143434,252\n-964.3040284401452,253\n-263.1808777708194,254\n-413.5803986409583,255\n-2.201068263763564,256\n-267.0071470771915,257\n-260.1376724372954,258\n-261.74101574274283,259\n-131.9105311243417,260\n-448.65855074975116,261\n-429.87923852170985,262\n-129.6193049217545,263\n-969.5628623509068,264\n-126.66474893150023,265\n-274.831349841074,266\n-129.83226454279134,267\n-383.679353672769,268\n-1052.0053869596522,269\n-272.1168027318256,270\n-1.427957214813859,271\n-2.740727605272145,272\n-3.116432645284185,273\n-1169.313384106829,274\n-791.0439997626511,275\n-1041.2273034436687,276\n-402.8287391137082,277\n-5.940200658240485,278\n-732.8929820086169,279\n-264.8407615955073,280\n-4.505348292019265,281\n-1188.5563444835811,282\n-963.778489959107,283\n-508.5885039093037,284\n-133.18251814941289,285\n-425.2095475052423,286\n-127.2187277019593,287\n-491.2177710740775,288\n-2.149364530985958,289\n-282.51505501219543,290\n-414.6238484536778,291\n-0.09329528015836464,292\n-0.34757233036448193,293\n-129.29797727013664,294\n-273.46557881457784,295\n-2.264203632734038,296\n-0.6221158578198953,297\n-137.1714059471168,298\n-1038.359679554627,299\n-280.317031815156,300\n-129.9130353809026,301\n-267.43475692245573,302\n-128.16816624847405,303\n-268.63122872419564,304\n-953.3902355984969,305\n-132.58473367449048,306\n-129.07391775354674,307\n-131.9533027951362,308\n-266.9613660961888,309\n-398.11443703741685,310\n-132.01569110778127,311\n-261.4524409861757,312\n-130.76038552754224,313\n-130.27758073904332,314\n-128.5328909226296,315\n-515.7664435846518,316\n-403.0755790744836,317\n-4.071167816415755,318\n-264.7810288102111,319\n-632.712305318958,320\n-683.431573567609,321\n-0.7438418075070443,322\n-0.05023690320918215,323\n-133.84361705007694,324\n-555.669044514544,325\n-129.1050539970185,326\n-131.37904854519508,327\n-1065.102541327549,328\n-125.21218687383916,329\n-265.4961028415644,330\n-128.93439860896171,331\n-2.5463065259661364,332\n-892.9061853235112,333\n-274.52581268030235,334\n-129.6623200558345,335\n-380.276942423357,336\n-128.6342763572376,337\n-129.36817330895622,338\n-820.4680424573669,339\n-4.814480144356802,340\n-130.03104481784547,341\n-131.40972674386987,342\n-132.43206472766053,343\n-131.10292909304857,344\n-142.12796213899267,345\n-1.2449715430215178,346\n-395.5971379532543,347\n-137.1032230160052,348\n-132.4153529771457,349\n-249.24718643239476,350\n-380.2983361998553,351\n-383.9927883000427,352\n-1013.1851779768176,353\n-266.5015626216723,354\n-131.78673607770514,355\n-267.6061234074856,356\n-405.85171049605947,357\n-258.02594417693354,358\n-520.8447508498335,359\n-134.63334367572654,360\n-419.236262001543,361\n-385.42191131678817,362\n-380.0541004465699,363\n-130.73528766230783,364\n-127.29132686260877,365\n-1047.650556810366,366\n-133.50850034218854,367\n-129.19179313884274,368\n-382.19760974673386,369\n-459.99344258576065,370\n-130.5772533835066,371\n-258.6915090979332,372\n-131.225993984309,373\n-2.307935564530814,374\n-381.0481201297432,375\n-263.13129723956706,376\n-128.38499437285228,377\n-128.91816804394875,378\n-402.16511915016144,379\n-428.4494441029423,380\n-274.4714786496533,381\n-463.4623132944519,382\n-127.29886848372509,383\n-129.96043351859282,384\n-128.64831889152399,385\n-253.58344915212325,386\n-132.8162915670369,387\n-261.93506456711094,388\n-262.39540227220493,389\n-260.8092114201002,390\n-129.43366071005198,391\n-0.5340992273939197,392\n-128.85111073510905,393\n-128.25477694966304,394\n-412.40810831862757,395\n-266.9218270456268,396\n-412.41758131506464,397\n-128.94429376414604,398\n-131.09949798355325,399\n-291.15923883350524,400\n-267.64527622386566,401\n-489.44021739425966,402\n-128.0549415277166,403\n-2.4489440262340567,404\n-403.8335832447348,405\n-410.8634983022809,406\n-127.15800890668305,407\n-428.3714272534157,408\n-137.91503165159457,409\n-429.3623718012684,410\n-267.128871121456,411\n-445.32875635963103,412\n-132.2639368673305,413\n-0.8292453044283625,414\n-132.19100503947476,415\n-127.83917746509681,416\n-130.37539777604732,417\n-271.6062650630438,418\n-264.4619258087007,419\n-131.37611987894263,420\n-131.59731181511395,421\n-531.1541840593379,422\n-262.17593588805056,423\n-131.28799968041162,424\n-131.08410689768246,425\n-132.6621370964813,426\n-286.2292489013363,427\n-131.6393376461004,428\n-307.00318017213027,429\n-428.77958288483137,430\n-130.58812017651263,431\n-357.6013304151711,432\n-926.5543613728378,433\n-131.11952790466398,434\n-260.9048452324567,435\n-134.46648463849618,436\n-264.3682002226004,437\n-0.3251696599549742,438\n-129.24645816162231,439\n-128.56839953980506,440\n-130.09193537825305,441\n-419.5039599799257,442\n-266.0338824811493,443\n-426.09314158246985,444\n-127.19881918845937,445\n-263.18838755032715,446\n-258.86289193906407,447\n-271.096836608607,448\n-389.65130588200174,449\n-1031.9517676001858,450\n-131.18981842282892,451\n-0.898636174689319,452\n-0.31304377708296094,453\n-133.75025266907096,454\n-404.99231506642457,455\n-394.11673240044604,456\n-464.8766583653142,457\n-132.28334086021638,458\n-136.11578944706451,459\n-266.5710215054984,460\n-132.0794712405045,461\n-426.6872296142505,462\n-0.40698260330357183,463\n-138.08564120788893,464\n-441.4021941759973,465\n-253.54833070632137,466\n-125.04346215500235,467\n-129.232547160193,468\n-129.59957341238032,469\n-131.58540041371165,470\n-0.11520531154230412,471\n-256.8222429188776,472\n-0.5340904204089344,473\n-0.11227800539307481,474\n-507.7925202898569,475\n-128.79352500896024,476\n-129.72501556935282,477\n-276.97311219905663,478\n-130.17404532044688,479\n-0.9855007735629081,480\n-274.73365370298256,481\n-264.1171658327858,482\n-0.6039651298632206,483\n-1.888962290308856,484\n-409.91178515945825,485\n-130.33859574210902,486\n-0.2526299082471999,487\n-0.1920064976020393,488\n-131.45519611727744,489\n-125.74396990213165,490\n-129.04985794559246,491\n-0.5514446130732259,492\n-370.824851015201,493\n-129.04752417159904,494\n-254.76937983878165,495\n-259.73307483465777,496\n-270.1910819389594,497\n-130.98638206437136,498\n-266.91815345927745,499\n-131.3358679564876,500\n-126.55290912236813,501\n-1.614728945340729,502\n-130.21371932527848,503\n-295.18125820398853,504\n-427.39651854717204,505\n-453.70126943626775,506\n-392.8612982565005,507\n-132.3081411598238,508\n-1.7547161753581573,509\n-126.67013570862441,510\n-442.2019227597015,511\n-133.66773407708973,512\n-133.69331730854222,513\n-131.0922238467658,514\n-133.01105763727813,515\n-131.96783001342584,516\n-279.9882998319758,517\n-413.736116709116,518\n-279.77571679128346,519\n-131.44542173039224,520\n-273.2931079674701,521\n-127.1623159788046,522\n-129.9227186861699,523\n-263.37337100433604,524\n-566.0529729813618,525\n-129.73312095522687,526\n-131.70386956436255,527\n-130.13161424610283,528\n-309.4025221242795,529\n-507.67091242658097,530\n-132.54272433386436,531\n-456.4901684756131,532\n-129.3366896838395,533\n-270.47630535374753,534\n-390.3088512357022,535\n-281.02620788897144,536\n-132.89269953948607,537\n-393.0647201624002,538\n-258.92109065066575,539\n-410.4206335127937,540\n-131.57547229048726,541\n-131.56327221156914,542\n-268.11553352451983,543\n-131.67056955537782,544\n-749.9462559889108,545\n-132.20753981710862,546\n-292.8537038715889,547\n-380.05751010362917,548\n-130.03996133883592,549\n-128.45062327971803,550\n-130.526224153602,551\n-1.0231352140755114,552\n-250.10961630919223,553\n-405.4763269431142,554\n-401.95020141073957,555\n-133.09880822529374,556\n-324.05008018357483,557\n-941.2818263786694,558\n-246.40594531279277,559\n-125.2998799402015,560\n-272.4818009764135,561\n-960.3997627433577,562\n-134.65460409721254,563\n-124.56823270745079,564\n-1.124726873996279,565\n-131.59965718764838,566\n-370.94146604689746,567\n-662.9994903141712,568\n-1.3445306641329808,569\n-269.3180595354585,570\n-318.31927787287253,571\n-132.51219852985957,572\n-130.9614660811535,573\n-126.56227444508643,574\n-408.4003489733563,575\n-408.94142143793306,576\n-384.43851232055715,577\n-1.508522255308678,578\n-281.32113219154826,579\n-273.63815386032684,580\n-1.1127182500268686,581\n-130.9937721719571,582\n-384.4660400346215,583\n-133.17450798020883,584\n-0.8380412523200846,585\n-129.60752421883006,586\n-2.965755327698331,587\n-127.41307150661144,588\n-129.1439589815816,589\n-278.00265016077816,590\n-130.79036283081433,591\n-0.8082693563482567,592\n-270.0810070446028,593\n-129.7888608255029,594\n-133.41426633254,595\n-275.90740486217214,596\n-130.45497775191708,597\n-264.526821885127,598\n-262.44742223034166,599\n-132.8136271364434,600\n-129.1296158272721,601\n-129.59003408143073,602\n-0.7752317939001201,603\n-442.62893536199675,604\n-260.01888100585677,605\n-128.85564276280897,606\n-2.0900013290340924,607\n-378.547545811925,608\n-0.45818804024405085,609\n-1.0427798030484328,610\n-131.14052170564514,611\n-131.0404965686141,612\n-127.41910769556478,613\n-289.0971815413776,614\n-307.0791071137063,615\n-137.77679722837084,616\n-300.30632660038896,617\n-252.7329259241285,618\n-267.46033876325237,619\n-128.3586525759139,620\n-1.0066521250498044,621\n-1.099981281258873,622\n-135.5629644873091,623\n-131.41428592344133,624\n-269.7473321652102,625\n-126.29426151982169,626\n-0.20491643914740534,627\n-287.3241431616045,628\n-261.84258758840065,629\n-430.62183648619424,630\n-377.82881130032905,631\n-131.67625168686934,632\n-130.42295892662074,633\n-128.69320934103726,634\n-261.16949289836685,635\n-424.9102414171337,636\n-128.14010277388067,637\n-131.87650704206425,638\n-1.5059015557737219,639\n-128.07212356043266,640\n-266.8406437015608,641\n-383.66185663655807,642\n-130.88187567904524,643\n-261.69653263903524,644\n-266.34886911279074,645\n-264.50521677454736,646\n-128.0223013013965,647\n-394.60245443239273,648\n-131.36040823277818,649\n-130.57095870503105,650\n-0.23693405918708585,651\n-433.94029691817644,652\n-0.05798460458972421,653\n-1.32311592011254,654\n-265.12372357068637,655\n-127.20326134213363,656\n-127.70693435614356,657\n-0.5383526683091888,658\n-314.0993106550805,659\n-129.0754615721328,660\n-131.3440461957394,661\n-267.8609514121298,662\n-127.2252629500192,663\n-782.0185158873359,664\n-132.73194440056224,665\n-472.61491079730166,666\n-0.5681079544814104,667\n-0.7525942307751928,668\n-129.78856116167506,669\n-258.7399151358298,670\n-261.35697450429484,671\n-0.09969538543128705,672\n-128.92855577564433,673\n-130.9883985880059,674\n-129.3127318681393,675\n-135.34002847472516,676\n-130.47067884103015,677\n-256.35679754772434,678\n-130.48491831442837,679\n-131.88397892631497,680\n-130.6919272898225,681\n-3.51628905538713,682\n-126.92262328518643,683\n-438.1725198302756,684\n-282.9683176239621,685\n-281.4174175718277,686\n-388.6991929258608,687\n-2.259261642301432,688\n-905.7166686013921,689\n-255.14629295843554,690\n-123.31121969399854,691\n-124.83108288347827,692\n-128.4830824684178,693\n-269.24868604129995,694\n-404.17449365983884,695\n-1081.3754450600861,696\n-130.3135477885459,697\n-922.9155544307238,698\n-131.3127034561507,699\n-3.5876478243828354,700\n-263.40186920941113,701\n-478.13746813609384,702\n-250.24834350506987,703\n-131.4551655252192,704\n-1086.0208089295388,705\n-379.4303433144596,706\n-1133.9462847272978,707\n-1203.615119418819,708\n-1178.0355063968727,709\n-131.70012170695864,710\n-132.73079904403303,711\n-126.91413454423446,712\n-0.04255804803516021,713\n-131.88345032848696,714\n-1.6908764548556243,715\n-1173.8472835652422,716\n-130.3943619026555,717\n-134.2791045851467,718\n-387.5557550695827,719\n-0.5567651226787648,720\n-274.4160741881581,721\n-265.915300458194,722\n-356.55139261823297,723\n-123.03461965027455,724\n-266.4792008585232,725\n-1202.7035577540319,726\n-266.30961317292116,727\n-1104.4029141281846,728\n-129.06300969511244,729\n-1096.2911548626391,730\n-264.851079856607,731\n-1181.5393527564322,732\n-3.654189476382843,733\n-303.6976661927392,734\n-132.12107596759802,735\n-131.9250892442401,736\n-270.7562628830797,737\n-2.690915122388185,738\n-1088.3889010176765,739\n-133.23651541997253,740\n-2.889429499730143,741\n-1123.5532070254262,742\n-309.0093106627237,743\n-258.55008137880515,744\n-1153.6138570434093,745\n-1110.4133701707415,746\n-333.15813355747514,747\n-136.0353128231204,748\n-1126.4536346174525,749\n-999.0250732655518,750\n-9.140735241098838,751\n-137.81412142137435,752\n-134.17916606104674,753\n-260.78422100227925,754\n-1102.9150558338663,755\n-1092.4080672035927,756\n-132.83656039154593,757\n-136.23781948044964,758\n-412.13091844079605,759\n-255.7938298942125,760\n-1116.3353458779018,761\n-288.64394129376655,762\n-133.2895396636901,763\n-272.2162380167979,764\n-129.47540884308205,765\n-136.7935699011869,766\n-1095.4190603302525,767\n-5.03387850443279,768\n-273.1973711475142,769\n-1063.7067161956859,770\n-397.93003935124113,771\n-5.6960426519009975,772\n-136.0395744703336,773\n-132.05325721070932,774\n-7.66858855602384,775\n-1180.8639695588965,776\n-10.10376234972853,777\n-131.2648362858944,778\n-265.98120746422927,779\n-4.515961398550995,780\n-269.6686707341921,781\n-1072.9772011659707,782\n-255.7002583482052,783\n-276.12515488805883,784\n-5.3992273969413205,785\n-394.0040863771805,786\n-252.8098589821395,787\n-298.8156858293448,788\n-259.49054943443673,789\n-131.8624551053748,790\n-134.58918256967218,791\n-4.480344721787044,792\n-276.990020174181,793\n-127.83941177131827,794\n-1058.547271414725,795\n-6.384070511268124,796\n-4.063058426614309,797\n-131.40613289906966,798\n-384.9437571918901,799\n-265.8011210098687,800\n-387.2251561450778,801\n-131.73910027444754,802\n-258.7766355879836,803\n-131.69479971508574,804\n-441.44504953416845,805\n-271.8490314758892,806\n-8.194349601906648,807\n-129.2797289436157,808\n-426.8703919091574,809\n-259.5002920384034,810\n-130.81236154474888,811\n-128.61990614726906,812\n-133.76488529838446,813\n-134.6812742953968,814\n-291.0891497661674,815\n-387.5499384209569,816\n-389.3477837178997,817\n-274.5928964309988,818\n-269.78914510916195,819\n-359.9093856266961,820\n-134.58532892959084,821\n-5.546399310698313,822\n-128.67536501786955,823\n-381.8971847668165,824\n-272.1623466516663,825\n-276.51515148178277,826\n-4.638946421798701,827\n-416.38515236358086,828\n-135.85545502890096,829\n-268.47613294771736,830\n-475.8096674474275,831\n-416.86735005237165,832\n-2.8908309481199663,833\n-134.12099077247595,834\n-3.2591793206950213,835\n-136.712066651335,836\n-269.92521208632616,837\n-4.292612229859982,838\n-310.0431269510179,839\n-133.76510542833927,840\n-133.03887035200253,841\n-266.78829412790753,842\n-132.94675990154516,843\n-268.4806113509835,844\n-133.25465814991787,845\n-3.2578734958959745,846\n-268.7982747796157,847\n-382.2689688134644,848\n-276.8215889075903,849\n-134.8297846334945,850\n-272.4051750055764,851\n-3.0994043613618483,852\n-257.74542499363486,853\n-136.70117264406105,854\n-5.927344126736666,855\n-386.48042976630325,856\n-269.2158866432635,857\n-462.58870921168096,858\n-131.70116780222796,859\n-388.37808714189543,860\n-131.66718040184844,861\n-133.390187975207,862\n-2.546677168281118,863\n-130.2353523518926,864\n-404.6752078259321,865\n-134.4221125747506,866\n-132.5784358626518,867\n-278.0465708988738,868\n-266.1954702893785,869\n-136.6783306387619,870\n-459.4578832004511,871\n-126.31254028625452,872\n-264.2888255788324,873\n-398.3964860782316,874\n-271.44670056156684,875\n-391.62175214688847,876\n-133.78249426888257,877\n-131.02046606534753,878\n-282.7301849197012,879\n-664.4094110555355,880\n-266.7922140376474,881\n-272.03118650071815,882\n-3.2383541369414575,883\n-400.473906069943,884\n-271.9514887901532,885\n-130.7241539787133,886\n-132.75943831123078,887\n-295.85006812002376,888\n-264.5708665183664,889\n-606.2960639148018,890\n-267.70656572577616,891\n-268.7317800284326,892\n-382.01770286057143,893\n-269.5697197666444,894\n-387.47448195470463,895\n-131.36531631971928,896\n-128.72438324393363,897\n-2.3909132272115077,898\n-3.4280611696906336,899\n-578.6839695365671,900\n-136.28294537913303,901\n-397.82938952608407,902\n-268.43234207129393,903\n-134.94748886024055,904\n-3.561368440349063,905\n-269.0690765671258,906\n-3.7574911893235594,907\n-411.6410970138961,908\n-448.79888303813505,909\n-687.4298394076347,910\n-443.21777503110826,911\n-4.29535197204481,912\n-465.5747426895258,913\n-268.30436330071666,914\n-130.16750861120164,915\n-257.6342703934387,916\n-5.049090368459385,917\n-5.3932129946318454,918\n-275.445674099593,919\n-265.9595358281853,920\n-511.4614452705677,921\n-273.4488546428298,922\n-391.39146730651953,923\n-132.72036097704634,924\n-131.55177488856228,925\n-136.0992494007178,926\n-267.56452624648546,927\n-4.371184325015422,928\n-4.565721933621111,929\n-4.674460659297859,930\n-517.8856753710986,931\n-413.45465081641686,932\n-390.25598993877117,933\n-263.96030387341665,934\n-417.8961437832366,935\n-136.48775699990074,936\n-448.0022289517615,937\n-5.603592644433703,938\n-267.5567854439665,939\n-4.867449906615465,940\n-141.69471424962785,941\n-139.56812340065056,942\n-441.44775388364496,943\n-387.9900799517474,944\n-274.42404221952546,945\n-475.52095215826336,946\n-3.99564273378651,947\n-410.302953976846,948\n-138.31342888652594,949\n-390.5958583783888,950\n-402.5777507175182,951\n-486.11601105917185,952\n-3.313305080130222,953\n-277.90384979400005,954\n-420.8001956907993,955\n-3.488641265910626,956\n-390.7156671616855,957\n-644.615260478683,958\n-134.33064543152432,959\n-3.5679845633150578,960\n-545.0406411918901,961\n-590.4013886325864,962\n-132.95410084112024,963\n-267.43302227003255,964\n-401.4782694928446,965\n-542.4413959428059,966\n-267.266488341028,967\n-273.9666631235666,968\n-129.97656382742247,969\n-382.86667879352433,970\n-132.10886586517077,971\n-487.03544751745636,972\n-265.8180387226017,973\n-557.782148396507,974\n-438.79170965592795,975\n-131.26225218994483,976\n-131.63790458977772,977\n-131.66951920627508,978\n-275.780275837175,979\n-392.3742321726265,980\n-132.91123699478314,981\n-1.8899226623661962,982\n-125.86713417720703,983\n-386.78524955307705,984\n-263.5119139434558,985\n-264.8935457442949,986\n-129.6201111530344,987\n-270.40201848156346,988\n-551.027255331022,989\n-268.94508202985685,990\n-268.07808675804654,991\n-3.39025817554275,992\n-675.818545839607,993\n-1.870275555996001,994\n-558.2524650547359,995\n-129.69524060101458,996\n-681.0008154753771,997\n-661.4876096124401,998\n-265.40732207607306,999\n" } ]