[
  {
    "path": "README.md",
    "content": "# Codebase for \"Time-series prediction\" with RNN, GRU, LSTM and Attention\n\nAuthors: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n\nThis directory contains implementations of basic time-series prediction\nusing RNN, GRU, LSTM or Attention methods.\nTo run the pipeline, simply run python3 -m main_time_series_prediction.py.\n\nNote: We recommend to do MinMax normalization on both input and output.\n\n## Stages of time-series prediction framework:\n\n-   Load dataset (Google stocks data)\n-   Train model:\n    (1) RNN based: Simple RNN, GRU, LSTM\n    (2) Attention based\n-   Evaluate the performance: MAE or MSE metrics\n\n### Command inputs:\n\n-   train_rate: training data ratio\n-   seq_len: sequence length\n-   task: classification or regression\n-   model_type: rnn, lstm, gru, or attention\n-   h_dim: hidden state dimensions\n-   n_layer: number of layers\n-   batch_size: the number of samples in each mini-batch\n-   epoch: the number of iterations\n-   learning_rate: learning rates\n-   metric_name: mse or mae\n\n### Example command\n\n```shell\n$ python3 main_time_series_prediction.py \n--train_rate 0.8 --seq_len 7 --task regression --model_type lstm\n--h_dim 10 --n_layer 3 --batch_size 32 --epoch 100 --learning_rate 0.01\n--metric_name mae\n```\n\n### Outputs\n\n-   MAE or MSE performance of trained model\n"
  },
  {
    "path": "basic_attention.py",
    "content": "\"\"\"Basic Attention core functions for time-series prediction.\n\nAuthor: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n\"\"\"\n\n# Necessary packages\nimport tensorflow as tf\nimport numpy as np\nimport os\nimport shutil\n\n\nclass Attention():\n  \"\"\"Attention class.\n  \n  Attributes:\n    - model_parameters:\n      - task: classificiation or regression\n      - h_dim: hidden state dimensions\n      - batch_size: the number of samples in each mini-batch\n      - epoch: the number of iterations\n      - learning_rate: learning rate of training\n  \"\"\"\n  def __init__(self, model_parameters):\n\n    tf.compat.v1.reset_default_graph()\n    self.task = model_parameters['task']\n    self.h_dim = model_parameters['h_dim']\n    self.batch_size = model_parameters['batch_size']\n    self.epoch = model_parameters['epoch']\n    self.learning_rate = model_parameters['learning_rate']\n    \n    self.save_file_directory = 'tmp/attention/'\n    \n    \n  def process_batch_input_for_RNN(self, batch_input):\n    \"\"\"Function to convert batch input data to use scan ops of tensorflow.\n    \n    Args:\n      - batch_input: original batch input\n    \n    Returns:\n      - x: batch_input for RNN \n    \"\"\"\n    batch_input_ = tf.transpose(batch_input, perm=[2, 0, 1])\n    x = tf.transpose(batch_input_)\n    return x\n\n\n  def sample_X(self, m, n):\n    \"\"\"Sample from the real data (Mini-batch index sampling).\n    \"\"\"\n    return np.random.permutation(m)[:n]  \n  \n  \n  def fit(self, x, y):\n    \"\"\"Train the model.\n    \n    Args:\n      - x: training feature\n      - y: training label\n    \"\"\"\n        \n    # Basic parameters\n    no, seq_len, x_dim = x.shape\n    y_dim = len(y[0, :])\n    \n    # Weights for GRU\n    Wr = tf.Variable(tf.zeros([x_dim, self.h_dim]))\n    Ur = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))\n    br = tf.Variable(tf.zeros([self.h_dim]))\n        \n    Wu = tf.Variable(tf.zeros([x_dim, self.h_dim]))\n    Uu = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))\n    bu = tf.Variable(tf.zeros([self.h_dim]))\n        \n    Wh = tf.Variable(tf.zeros([x_dim, self.h_dim]))\n    Uh = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))\n    bh = tf.Variable(tf.zeros([self.h_dim]))\n                \n    # Weights for attention mechanism \n    Wa1 = tf.Variable(tf.random.truncated_normal([self.h_dim + x_dim, \n                                                  self.h_dim], \n                                                 mean=0, stddev=.01))\n    Wa2 = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], \n                                                 mean=0, stddev=.01))\n    ba1 = tf.Variable(tf.random.truncated_normal([self.h_dim], \n                                                 mean=0, stddev=.01))\n    ba2 = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01))\n            \n    # Weights for output layers\n    Wo = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], \n                                         mean=0, stddev=.01))\n    bo = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01))\n    \n    # Target\n    Y = tf.compat.v1.placeholder(tf.float32, [None,1])    \n    # Input vector with shape[batch, seq, embeddings]\n    _inputs = tf.compat.v1.placeholder(tf.float32, shape=[None, None, x_dim], \n                                       name='inputs')\n    \n    # Processing inputs to work with scan function\n    processed_input = self.process_batch_input_for_RNN(_inputs)\n            \n    # Initial Hidden States\n    initial_hidden = _inputs[:, 0, :]\n    initial_hidden = tf.matmul(initial_hidden, tf.zeros([x_dim, self.h_dim]))\n        \n \n    def GRU(previous_hidden_state, x):\n      \"\"\"Function for Forward GRU cell.\n      \n      Args:\n        - previous_hidden_state\n        - x: current input\n        \n      Returns:\n        - current_hidden_state\n      \"\"\"\n      # R Gate\n      r = tf.sigmoid(tf.matmul(x, Wr) + \\\n                     tf.matmul(previous_hidden_state, Ur) + br)\n      # U Gate\n      u = tf.sigmoid(tf.matmul(x, Wu) + \\\n                     tf.matmul(previous_hidden_state, Uu) + bu)\n      # Final Memory cell\n      c = tf.tanh(tf.matmul(x, Wh) + \\\n                  tf.matmul( tf.multiply(r, previous_hidden_state), Uh) + bh)\n      # Current Hidden state\n      current_hidden_state = tf.multiply( (1 - u), previous_hidden_state ) + \\\n                             tf.multiply( u, c )\n      return current_hidden_state\n        \n    \n    def get_states():\n      \"\"\"Function to get the hidden and memory cells after forward pass.\n      \n      Returns:\n        - all_hidden_states\n      \"\"\"\n      # Getting all hidden state through time\n      all_hidden_states = tf.scan(GRU, processed_input, \n                                  initializer=initial_hidden, name='states')\n      return all_hidden_states\n              \n        \n    def get_attention(hidden_state):\n      \"\"\"Function to get attention with the last input.\n      \n      Args:\n        - hidden_states\n        \n      Returns:\n        - e_values\n      \"\"\"\n      inputs = tf.concat((hidden_state, processed_input[-1]), axis = 1)\n      hidden_values = tf.nn.tanh(tf.matmul(inputs, Wa1) + ba1)\n      e_values = (tf.matmul(hidden_values, Wa2) + ba2)\n      return e_values\n        \n\n    def get_outputs():\n      \"\"\"Function for getting output and attention coefficient.\n      \n      Returns:\n        - output: final outputs\n        - a_values: attention values\n      \"\"\"\n      all_hidden_states = get_states()\n      all_attention = tf.map_fn(get_attention, all_hidden_states)\n      a_values = tf.nn.softmax(all_attention, axis = 0)\n      final_hidden_state = tf.einsum('ijk,ijl->jkl', a_values, \n                                     all_hidden_states)\n      output = tf.nn.sigmoid(tf.matmul(final_hidden_state[:,0,:], Wo) + bo, \n                             name='outputs')\n      return output, a_values   \n                \n    # Getting all outputs from rnn\n    outputs, attention_values = get_outputs()\n    \n    # reshape out for sequence_loss\n    if self.task == 'classification':\n      loss = tf.reduce_mean(Y * tf.log(outputs + 1e-8) + \\\n                            (1-Y) * tf.log(1-outputs + 1e-8))\n    elif self.task == 'regression':\n      loss = tf.sqrt(tf.reduce_mean(tf.square(outputs - Y)))\n    \n    # Optimization\n    optimizer = tf.compat.v1.train.AdamOptimizer(self.learning_rate)\n    train = optimizer.minimize(loss)\n\n    # Sessions\n    sess = tf.compat.v1.Session()\n    sess.run(tf.compat.v1.global_variables_initializer())\n        \n    # Training\n    iteration_per_epoch = int(no/self.batch_size)\n    iterations = int((self.epoch * no) / self.batch_size)\n    \n    for i in range(iterations):\n      \n      idx = self.sample_X(no, self.batch_size)\n      Input = x[idx,:,:]            \n      _, step_loss = sess.run([train, loss], \n                              feed_dict={Y: y[idx], _inputs: Input})\n                \n      # Print intermediate results\n      if i % iteration_per_epoch == iteration_per_epoch-1:\n        print('Epoch: ' + str(int(i/iteration_per_epoch)) + \n              ', Loss: ' + str(np.round(step_loss, 4)))\n        \n    # Reset the directory for saving\n    if not os.path.exists(self.save_file_directory):\n      os.makedirs(self.save_file_directory)\n    else:\n      shutil.rmtree(self.save_file_directory)\n  \n    # Save model\n    inputs = {'inputs': _inputs}\n    outputs = {'outputs': outputs}\n    tf.compat.v1.saved_model.simple_save(sess, self.save_file_directory, \n                                         inputs, outputs)    \n        \n       \n  def predict(self, test_x):\n    \"\"\"Prediction with trained model.\n    \n    Args:\n      - test_x: testing features\n      \n    Returns:\n      - test_y_hat: predictions on testing set\n    \"\"\"\n    \n    graph = tf.Graph()\n    with graph.as_default():\n      with tf.compat.v1.Session() as sess:\n        tf.compat.v1.saved_model.loader.load(sess, [tf.saved_model.SERVING], \n                                             self.save_file_directory)\n        x = graph.get_tensor_by_name('inputs:0')\n        outputs = graph.get_tensor_by_name('outputs:0')\n    \n        test_y_hat = sess.run(outputs, feed_dict={x: test_x})\n        \n    return test_y_hat\n    "
  },
  {
    "path": "basic_rnn_lstm_gru.py",
    "content": "\"\"\"General RNN core functions for time-series prediction.\n\nAuthor: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n\"\"\"\n\n# Necessary packages\nimport os\nimport tensorflow as tf\nimport numpy as np\nfrom datetime import datetime\nfrom tensorflow.keras import layers\nfrom keras.callbacks import ModelCheckpoint\nfrom utils import binary_cross_entropy_loss, mse_loss, rnn_sequential\n\n\nclass GeneralRNN():\n  \"\"\"RNN predictive model to time-series.\n  \n  Attributes:\n    - model_parameters:\n      - task: classification or regression\n      - model_type: 'rnn', 'lstm', or 'gru'\n      - h_dim: hidden dimensions\n      - n_layer: the number of layers\n      - batch_size: the number of samples in each batch\n      - epoch: the number of iteration epochs\n      - learning_rate: the learning rate of model training\n  \"\"\"\n\n  def __init__(self, model_parameters):\n\n    self.task = model_parameters['task']\n    self.model_type = model_parameters['model_type']\n    self.h_dim = model_parameters['h_dim']\n    self.n_layer = model_parameters['n_layer']\n    self.batch_size = model_parameters['batch_size']\n    self.epoch = model_parameters['epoch']\n    self.learning_rate = model_parameters['learning_rate']\n    \n    assert self.model_type in ['rnn', 'lstm', 'gru']\n\n    # Predictor model define\n    self.predictor_model = None\n\n    # Set path for model saving\n    model_path = 'tmp'\n    if not os.path.exists(model_path):\n      os.makedirs(model_path)\n    self.save_file_name = '{}'.format(model_path) + \\\n                          datetime.now().strftime('%H%M%S') + '.hdf5'\n  \n\n  def _build_model(self, x, y):\n    \"\"\"Construct the model using feature and label statistics.\n    \n    Args:\n      - x: features\n      - y: labels\n      \n    Returns:\n      - model: predictor model\n    \"\"\"    \n    # Parameters\n    h_dim = self.h_dim\n    n_layer = self.n_layer\n    dim = len(x[0, 0, :])\n    max_seq_len = len(x[0, :, 0])\n\n    model = tf.keras.Sequential()\n    model.add(layers.Masking(mask_value=0., input_shape=(max_seq_len, dim)))\n\n    for _ in range(n_layer - 1):\n      model = rnn_sequential(model, self.model_type, h_dim, return_seq=True)\n\n    model = rnn_sequential(model, self.model_type, h_dim, \n                           return_seq=False)\n    adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate, \n                                    beta_1=0.9, beta_2=0.999, amsgrad=False)\n\n    if self.task == 'classification':\n      model.add(layers.Dense(y.shape[-1], activation='sigmoid'))\n      model.compile(loss=binary_cross_entropy_loss, optimizer=adam)\n      \n    elif self.task == 'regression':\n      model.add(layers.Dense(y.shape[-1], activation='linear'))\n      model.compile(loss=mse_loss, optimizer=adam, metrics=['mse'])\n\n    return model\n  \n\n  def fit(self, x, y):\n    \"\"\"Fit the predictor model.\n    \n    Args:\n      - x: training features\n      - y: training labels\n      \n    Returns:\n      - self.predictor_model: trained predictor model\n    \"\"\"\n    idx = np.random.permutation(len(x))\n    train_idx = idx[:int(len(idx)*0.8)]\n    valid_idx = idx[int(len(idx)*0.8):]\n    \n    train_x, train_y = x[train_idx], y[train_idx]\n    valid_x, valid_y = x[valid_idx], y[valid_idx]\n    \n    self.predictor_model = self._build_model(train_x, train_y)\n\n    # Callback for the best model saving\n    save_best = ModelCheckpoint(self.save_file_name, monitor='val_loss',\n                                mode='min', verbose=False,\n                                save_best_only=True)\n\n    # Train the model\n    self.predictor_model.fit(train_x, train_y, \n                             batch_size=self.batch_size, epochs=self.epoch, \n                             validation_data=(valid_x, valid_y), \n                             callbacks=[save_best], verbose=True)\n\n    self.predictor_model.load_weights(self.save_file_name)\n    os.remove(self.save_file_name)\n\n    return self.predictor_model\n  \n  \n  def predict(self, test_x):\n    \"\"\"Return the temporal and feature importance.\n    \n    Args:\n      - test_x: testing features\n      \n    Returns:\n      - test_y_hat: predictions on testing set\n    \"\"\"\n    test_y_hat = self.predictor_model.predict(test_x)\n    return test_y_hat\n"
  },
  {
    "path": "data/google.csv",
    "content": "Open,High,Low,Volume,Close\n828.659973,833.450012,828.349976,1247700,831.659973\n823.02002,828.070007,821.655029,1597800,828.070007\n819.929993,824.400024,818.97998,1281700,824.159973\n819.359985,823,818.469971,1304000,818.97998\n819,823,816,1053600,820.450012\n816,820.958984,815.48999,1198100,819.23999\n811.700012,815.25,809.780029,1129100,813.669983\n809.51001,810.659973,804.539978,989700,809.559998\n807,811.840027,803.190002,1155300,808.380005\n803.98999,810.5,801.780029,1235200,806.969971\n799.700012,801.669983,795.25,1174200,801.340027\n802.98999,806,800.369995,1460400,801.48999\n793.799988,802.700012,792,1525800,798.530029\n799.679993,801.190002,791.190002,2023300,795.695007\n796.859985,801.25,790.52002,2143500,796.789978\n814.659973,815.840027,799.799988,3228900,802.320007\n834.710022,841.950012,820.440002,2951800,823.309998\n837.809998,838,827.01001,2734400,832.150024\n829.619995,835.77002,825.059998,1494500,835.669983\n822.299988,825.900024,817.820984,1461000,823.869995\n807.25,820.869995,803.73999,1901600,819.309998\n806.909973,806.909973,801.690002,1645000,805.02002\n805.119995,809.47998,801.799988,912000,802.174988\n805.809998,806.205017,800.98999,1293300,806.070007\n807.080017,807.140015,800.369995,1355800,804.609985\n807.47998,811.223999,806.690002,1090100,807.880005\n807.140015,807.390015,799.169983,1351000,806.359985\n805,808.150024,801.369995,1057900,807.909973\n807.859985,809.130005,803.51001,1174300,804.789978\n806.400024,809.966003,802.830017,1272400,806.650024\n795.26001,807.900024,792.203979,1620500,806.150024\n786.080017,794.47998,785.02002,1315400,794.02002\n788.359985,791.340027,783.159973,1065400,786.900024\n778.809998,789.630005,775.799988,1643100,786.140015\n782.75,782.780029,770.409973,1760200,771.820007\n783.330017,785.929993,778.919983,742200,782.789978\n793.700012,794.22998,783.200012,1132700,785.049988\n790.679993,797.859985,787.656982,789100,791.549988\n790.900024,792.73999,787.280029,623400,789.909973\n792.359985,793.320007,788.580017,969100,791.26001\n795.840027,796.676025,787.099976,1208700,794.559998\n796.76001,798.650024,793.27002,925100,796.419983\n790.219971,797.659973,786.27002,1225900,794.200012\n800.400024,800.856018,790.289978,2428300,790.799988\n797.340027,803,792.919983,1623700,797.849976\n797.400024,804,794.01001,1664500,797.070007\n793.900024,804.380005,793.340027,2119800,796.099976\n785.039978,791.25,784.35498,2097100,789.27002\n780,789.429993,779.020996,1780500,789.289978\n772.47998,778.179993,767.22998,1448200,776.419983\n761,771.359985,755.799988,1750800,771.190002\n764.72998,768.830017,757.340027,1632800,759.109985\n757.710022,763.900024,752.900024,1382900,762.52002\n744.590027,754,743.099976,1448700,750.5\n757.440002,759.849976,737.025024,2996900,747.919983\n770.070007,772.98999,754.830017,2365800,758.039978\n771.530029,778.5,768.23999,1604500,770.840027\n760,779.530029,759.799988,2172200,768.23999\n764.26001,765,760.52002,586000,761.679993\n767.72998,768.28302,755.25,1477400,760.98999\n772.630005,776.960022,767,1591800,768.27002\n762.609985,769.700012,760.599976,1323000,769.200012\n771.369995,775,760,1526200,760.539978\n766.919983,772.700012,764.22998,1287000,771.22998\n755.200012,766.359985,750.51001,1465200,764.47998\n746.969971,764.416016,746.969971,2341200,758.48999\n755.599976,757.849976,727.539978,3631700,736.080017\n756.539978,760.780029,750.380005,2394000,754.02002\n791.169983,791.169983,752.179993,4740100,762.559998\n779.940002,791.22699,771.669983,2602700,785.309998\n783.400024,795.632996,780.190002,1350800,790.51001\n774.5,785.190002,772.549988,1585100,782.52002\n750.659973,770.359985,750.560974,2134800,762.02002\n767.25,769.950012,759.030029,1943200,762.130005\n778.200012,781.650024,763.450012,1872400,768.700012\n782.890015,789.48999,775.539978,2406400,783.609985\n795.469971,796.859985,784,2427300,784.539978\n808.349976,815.48999,793.590027,4269900,795.369995\n801,803.48999,791.5,2749200,795.349976\n806.340027,806.97998,796.320007,1647700,799.070007\n816.679993,816.679993,805.140015,1576400,807.669983\n804.900024,815.179993,804.820007,1697500,813.109985\n795,799.5,794,1266200,799.369995\n803.299988,803.969971,796.030029,1757500,796.969971\n798.23999,804.599976,798.030029,1766800,801.5\n787.849976,801.609985,785.565002,2056900,795.26001\n779.799988,785.849976,777.5,1093000,779.960022\n781.650024,783.950012,776,852500,778.530029\n781.219971,781.219971,773,1365300,778.190002\n783.76001,788.130005,782.059998,937400,786.140015\n786.659973,792.280029,780.580017,1372500,783.070007\n777.710022,789.380005,775.869995,1174900,785.940002\n779.659973,779.659973,770.75,933200,775.080017\n779,780.47998,775.539978,1070700,776.859985\n779.309998,782.070007,775.650024,1461200,776.469971\n776.030029,778.710022,772.890015,1201400,776.429993\n774.25,776.065002,769.5,1278800,772.559998\n776.330017,780.940002,774.090027,1585300,777.289978\n781.440002,785.799988,774.231995,1314700,775.01001\n777.849976,781.809998,774.969971,1109800,781.559998\n775.5,785.98999,774.307983,1153200,783.01001\n782.73999,782.73999,773.070007,1533200,774.210022\n786.590027,788.929993,784.150024,1411900,786.900024\n780,789.849976,778.440002,1486200,787.210022\n772.659973,777.159973,768.301025,1167800,776.219971\n769,773.330017,768.530029,978600,771.409973\n772.419983,774,764.440979,1172800,765.700012\n769.75,769.75,764.659973,2049300,768.880005\n762.890015,773.799988,759.960022,1305100,771.76001\n759.609985,767.679993,759.109985,1087400,762.48999\n764.47998,766.219971,755.799988,1395000,759.690002\n755.130005,770.289978,754,1311000,769.02002\n770.099976,773.244995,759.659973,1885500,759.659973\n778.590027,780.349976,773.580017,1270300,775.320007\n780,782.72998,776.200012,893700,780.349976\n773.450012,782,771,1442800,780.080017\n773.01001,773.919983,768.409973,1072700,771.460022\n769.25,771.02002,764.299988,925100,768.780029\n767.01001,769.090027,765.380005,1248600,767.049988\n769.330017,774.466003,766.840027,1130000,769.090027\n768.73999,774.98999,766.61499,847600,772.150024\n769,776.080017,765.849976,1166700,769.539978\n767,771.890015,763.184998,926900,769.409973\n770.580017,774.5,767.070007,1072000,769.640015\n775.47998,776.440002,771.784973,928200,772.080017\n773.27002,774.539978,770.049988,951400,772.150024\n775,777.099976,773.130005,861500,775.419983\n780.01001,782.859985,777,719400,777.5\n777.320007,780.809998,773.530029,924200,779.909973\n780.299988,780.97998,773.44397,1028000,777.140015\n783.75,787.48999,780.109985,938200,782.440002\n781.5,783.39502,780.400024,740500,783.219971\n785,789.75,782.969971,975100,784.849976\n783.75,786.812012,782.778015,786400,784.679993\n781.099976,788.940002,780.570007,1318900,784.26001\n782,782.630005,778.091003,1107900,781.76001\n773.780029,783.039978,772.340027,1801200,782.219971\n772.219971,774.070007,768.794983,1140300,771.609985\n767.179993,773.210022,766.820007,1287400,773.179993\n768.690002,775.840027,767.849976,1784500,771.070007\n761.090027,780.429993,761.090027,2700500,772.880005\n772.710022,778.549988,766.77002,3841500,768.789978\n747.039978,748.650024,739.299988,3530200,745.909973\n738.280029,744.460022,737,1512500,741.77002\n739.039978,741.690002,734.27002,1186700,738.419983\n740.669983,742.609985,737.5,1032400,739.77002\n741.859985,743.23999,736.559998,1259800,742.73999\n740.359985,741.690002,735.830994,1026300,738.630005\n737.330017,742.130005,737.099976,1289700,741.190002\n729.890015,736.98999,729,1227500,736.960022\n722.710022,736.130005,721.190002,1295500,733.780029\n725.72998,725.73999,719.054993,1279300,719.849976\n721.580017,722.210022,718.030029,950200,720.950012\n723.619995,724,716.849976,935900,716.97998\n719.119995,722.940002,715.909973,1336900,720.640015\n708.049988,716.51001,707.23999,1111800,715.090027\n699.5,705.710022,696.434998,1575200,705.630005\n698.080017,698.200012,688.215027,1304200,695.359985\n689.97998,701.679993,689.090027,1411900,697.77002\n696.059998,696.940002,688.880005,1462600,694.950012\n692.200012,700.650024,692.130005,1344700,699.210022\n685.469971,692.320007,683.650024,1597700,692.099976\n683,687.429016,681.409973,1932600,684.109985\n678.969971,680.330017,673,2173800,680.039978\n671,672.299988,663.283997,2629000,668.26001\n675.169983,689.400024,673.450012,4449000,675.219971\n697.450012,701.950012,687,2171400,701.869995\n699.059998,700.859985,693.08197,1184300,697.460022\n698.400024,702.77002,692.01001,1465600,695.940002\n698.77002,702.47998,693.409973,2082500,693.710022\n708.650024,708.820007,688.452026,3402400,691.719971\n714.909973,716.650024,703.26001,1982500,710.359985\n719,722.97998,717.309998,1214500,718.919983\n716.47998,722.469971,713.119995,1306100,718.27002\n716.51001,725.440002,716.51001,1258900,718.359985\n719.469971,725.890015,716.429993,1216400,719.409973\n722.869995,729.539978,722.335999,988900,728.580017\n723.960022,728.570007,720.580017,1583700,728.280029\n719.840027,721.97998,716.549988,1336200,716.650024\n724.909973,724.909973,714.609985,1565300,716.549988\n729.27002,729.48999,720.559998,1226300,722.340027\n732.5,733.02002,724.169983,1341800,730.400024\n734.530029,737.210022,730.659973,1253600,734.150024\n731.73999,739.72998,731.26001,2129500,735.719971\n724.01001,733.935974,724,1974000,732.659973\n722.869995,728.330017,720.280029,1576300,724.119995\n720.76001,727.51001,719.705017,1629200,725.27002\n706.859985,720.969971,706.859985,1929500,720.090027\n706.530029,711.478027,704.179993,1330700,704.23999\n701.619995,714.580017,700.52002,1828400,709.73999\n702.359985,706,696.799988,1670200,700.320007\n703.669983,711.599976,700.630005,1766800,706.630005\n715.98999,721.52002,704.109985,2001200,706.22998\n709.130005,718.47998,705.650024,1317100,716.48999\n711.929993,716.661987,709.26001,1314500,710.830017\n717.059998,719.25,709,1360700,713.309998\n723.409973,724.47998,712.799988,1692100,715.289978\n716.75,723.5,715.719971,1569600,723.179993\n712,718.710022,710,1510300,712.900024\n698.380005,711.859985,698.106995,1826100,711.119995\n697.700012,702.320007,695.719971,1683500,701.429993\n690.48999,699.75,689.01001,1693500,695.700012\n696.869995,697.840027,692,1543800,692.359985\n697.630005,700.640015,691,1645300,698.210022\n690.700012,697.619995,689,2487700,693.01001\n708.26001,714.169983,689.549988,2867300,691.02002\n707.289978,708.97998,692.36499,3098600,705.840027\n725.419983,725.765991,703.026001,2744600,708.140015\n716.099976,723.929993,715.590027,1959200,723.150024\n726.299988,736.119995,713.609985,5951900,718.77002\n755.380005,760.450012,749.549988,3060500,759.140015\n758,758.132019,750.01001,1529200,752.669983\n769.51001,769.900024,749.330017,2030500,753.929993\n760.460022,768.049988,757.299988,1556000,766.609985\n753.97998,761,752.69397,1809300,759\n754.01001,757.309998,752.705017,1135300,753.200012\n749.159973,754.380005,744.260986,1707100,751.719971\n738,743.830017,731.01001,1353000,743.090027\n743.02002,745,736.049988,1220100,736.099976\n743.969971,745.450012,735.549988,1290800,739.150024\n745.369995,747,736.280029,1453200,740.280029\n735.77002,746.23999,735.559998,1053700,745.690002\n738,742.799988,735.369995,1132300,737.799988\n750.059998,752.799988,742.429993,1134200,745.289978\n738.599976,750.340027,737,1576700,749.909973\n749.25,750.849976,740.940002,1718800,744.950012\n750.099976,757.880005,748.73999,1782400,750.530029\n734.590027,747.25,728.76001,1903800,744.77002\n736.789978,738.98999,732.5,1301300,733.530029\n732.01001,737.747009,731,1594900,735.299988\n742.359985,745.719971,736.150024,1432100,738.059998\n737.460022,745,737.460022,1269700,740.75\n736.5,742.5,733.515991,1836500,742.090027\n741.859985,742,731.830017,2980700,737.599976\n736.450012,743.070007,736,1860800,737.780029\n726.369995,737.469971,724.51001,1624400,736.090027\n726.919983,732.289978,724.77002,1721000,728.330017\n726.809998,735.5,725.150024,1718300,730.48999\n720,726.919983,717.125,1970800,726.820007\n708.119995,716.440002,703.359985,2833500,712.820007\n698.469971,705.679993,694,1421500,705.23999\n688.590027,703.789978,685.340027,2076300,693.969971\n706.900024,708.091003,686.900024,2985100,695.159973\n714.98999,716.48999,706.02002,1972100,710.890015\n718.679993,719.450012,706.02002,1958000,712.419983\n719,720,712,1629000,718.849976\n703.619995,718.809998,699.77002,2151400,718.809998\n700.320007,710.890015,697.679993,2481100,697.77002\n708.580017,713.429993,700.859985,2243500,705.070007\n700.01001,705.97998,690.585022,1642200,705.75\n688.919983,700,680.780029,1963600,699.559998\n701.450012,708.400024,693.580017,2009300,695.849976\n707.450012,713.23999,702.51001,1949800,706.460022\n695.030029,703.080994,694.049988,1589300,700.909973\n710,712.349976,696.030029,1883200,697.349976\n698.090027,709.75,691.380005,2492600,708.400024\n692.97998,698,685.049988,2520000,691\n690.26001,693.75,678.599976,2141400,682.400024\n675,689.349976,668.867981,3024000,683.109985\n686.859985,701.309998,682.130005,2638000,684.119995\n672.320007,699.900024,668.77002,3608900,678.109985\n667.849976,684.030029,663.059998,4247400,682.73999\n703.869995,703.98999,680.150024,5105700,683.570007\n722.809998,727,701.859985,5168700,708.01001\n770.219971,774.5,720.5,6171000,726.950012\n784.5,789.869995,764.650024,6348100,764.650024\n750.460022,757.859985,743.27002,5139200,752\n731.530029,744.98999,726.799988,3474300,742.950012\n722.219971,733.690002,712.349976,2676400,730.960022\n713.669983,718.234985,694.390015,2194200,699.98999\n713.849976,718.280029,706.47998,1331700,713.039978\n723.580017,729.679993,710.01001,1711700,711.669983\n723.599976,728.130005,720.120972,2011800,725.25\n702.179993,719.190002,694.460022,2412200,706.590027\n688.609985,706.849976,673.26001,3445000,698.450012\n703.299988,709.97998,693.409973,2268100,701.789978\n692.289978,706.73999,685.369995,3592400,694.450012\n705.380005,721.924988,689.099976,2225800,714.719971\n730.849976,734.73999,698.609985,2501700,700.559998\n721.679993,728.75,717.317017,2024500,726.070007\n716.609985,718.85498,703.539978,2090600,716.030029\n731.450012,733.22998,713,2450900,714.469971\n730.309998,738.5,719.059998,2963700,726.390015\n730,747.179993,728.919983,1947000,743.619995\n746.450012,752,738.640015,1950700,742.580017\n743,744.059998,731.257996,3272800,741.840027\n769.5,769.5,758.340027,1489600,758.880005\n776.599976,777.599976,766.900024,1293300,771\n766.690002,779.97998,766.429993,1765000,776.599976\n752.919983,762.98999,749.52002,1515300,762.51001\n749.549988,751.349976,746.619995,527200,748.400024\n753.469971,754.210022,744,1565900,750.309998\n751.650024,754.849976,745.530029,1365400,750\n746.130005,750,740,1525700,747.77002\n746.51001,754.130005,738.150024,3148700,739.309998\n762.419983,762.679993,749,1553400,749.429993\n750,760.590027,739.434998,1993300,758.090027\n753,758.080017,743.01001,2666200,743.400024\n741.789978,748.72998,724.169983,2412500,747.77002\n741.159973,745.710022,736.75,2224400,738.869995\n752.849976,755.849976,743.830017,1984900,749.460022\n759.169983,764.22998,737.000977,2700000,751.609985\n757.890015,764.799988,754.200012,1829500,762.369995\n767.77002,768.72998,755.090027,1812300,763.25\n753.099976,768.48999,750,2757300,766.809998\n766.01001,768.994995,745.630005,2590600,752.539978\n768.900024,775.955017,758.960022,2230400,762.380005\n747.109985,768.950012,746.700012,2134600,767.039978\n748.809998,754.929993,741.27002,2097600,742.599976\n748.460022,753.409973,747.48999,838500,750.26001\n748.140015,752,746.059998,1122100,748.150024\n752,755.278992,737.630005,2333100,748.280029\n757.450012,762.708008,751.820007,1414500,755.97998\n746.530029,757.919983,743,2212300,756.599976\n738.73999,742,737.429993,1327100,738.409973\n727.580017,741.409973,727,1684300,740\n729.289978,731.844971,723.026978,1510900,725.299988\n715.599976,729.48999,711.330017,1905900,728.960022\n729.169983,731.150024,716.72998,2075500,717\n731,737.799988,728.64502,1837200,731.22998\n732.460022,741,730.22998,1366400,735.400024\n724.400024,730.590027,718.5,1608000,728.320007\n730.200012,734.710022,719.429993,2069800,724.890015\n731.5,735.409973,727.01001,1511600,733.76001\n729.469971,739.47998,729.469971,1861600,731.25\n722,733.099976,721.900024,1706700,728.109985\n718.859985,724.650024,714.719971,1565400,722.159973\n711.059998,721.619995,705.849976,1886300,721.109985\n715.72998,718,710.049988,1908800,710.809998\n710.5,718.26001,710.01001,1456000,716.919983\n707.330017,712.97998,703.080017,2178900,712.950012\n707.380005,713.619995,704.549988,2245800,708.48999\n701.549988,719.150024,701.26001,2716600,712.780029\n727.5,730,701.5,6653900,702\n646.700012,657.799988,644.01001,4071000,651.789978\n654.150024,655.869995,641.72998,1791100,642.609985\n664.039978,664.719971,644.195007,2490000,650.280029\n661.179993,666.820007,659.580017,1477300,666.099976\n664.109985,664.969971,657.200012,1611100,662.200012\n654.659973,663.130005,654.460022,1885700,661.73999\n653.210022,659.390015,648.849976,1415500,651.159973\n643.150024,657.812012,643.150024,1807700,652.299988\n642.090027,648.5,639.01001,1275200,646.669983\n640,645.98999,635.317993,1648700,643.609985\n641.359985,644.450012,625.559998,2182100,639.159973\n649.23999,650.609009,632.150024,2092700,642.359985\n638.840027,649.25,636.530029,2166300,645.440002\n632,643.01001,627,1803600,641.469971\n607.200012,627.340027,603.130005,2684800,626.909973\n608.369995,612.090027,599.849976,1867600,611.289978\n603.280029,608.76001,600.72998,2413400,608.419983\n597.280029,605,590.219971,2309500,594.969971\n610.340027,614.60498,589.380005,3127700,594.890015\n629.77002,629.77002,611,2174000,611.969971\n616.640015,627.320007,612.400024,2240100,625.799988\n622.049988,628.929993,620,1470900,622.359985\n627,627.549988,615.429993,2562900,622.690002\n634.400024,636.48999,625.940002,1788500,635.440002\n636.789978,640,627.02002,5133400,629.25\n637.789978,650.900024,635.02002,2274700,642.900024\n635.469971,637.950012,632.320007,1286500,635.97998\n626.700012,638.700012,623.780029,2084400,635.140015\n625.700012,625.859985,619.429993,1702300,623.23999\n619.75,625.780029,617.419983,1373500,625.77002\n613.099976,624.159973,611.429993,1905300,621.349976\n621.219971,626.52002,609.599976,1702100,612.719971\n612.48999,616.309998,604.119995,2279500,614.659973\n600,603.469971,595.25,2089000,600.700012\n617,619.710022,602.820984,1759600,606.25\n605.590027,614.340027,599.710022,2575600,614.340027\n602.359985,612.859985,594.099976,3702100,597.789978\n627.539978,635.799988,617.679993,2176700,618.25\n632.820007,636.880005,624.559998,1978700,630.380005\n639.400024,643.590027,622,3491300,637.609985\n610.349976,631.710022,599.049988,4235900,628.619995\n614.909973,617.450012,581.109985,3538000,582.059998\n573,599.330017,565.049988,5770300,589.609985\n639.780029,640.049988,612.330017,4265200,612.47998\n655.460022,662.98999,642.900024,2855300,646.830017\n656.599976,667,654.190002,2131600,660.900024\n661.900024,664,653.460022,1456100,656.130005\n656.799988,661.380005,651.23999,1051700,660.869995\n655.01001,659.85498,652.659973,1072100,657.119995\n659.322021,664.5,651.661011,1810700,656.450012\n663.080017,665,652.289978,2936700,659.559998\n669.200012,674.900024,654.27002,5029200,660.780029\n639.47998,643.440002,631.249023,1809200,633.72998\n640.22998,642.679993,629.710022,1403900,635.299988\n645,645.379028,632.25,1572600,642.679993\n634.330017,647.859985,633.159973,2334300,643.780029\n628.419983,634.809998,627.159973,1490900,629.25\n625.340027,633.05603,625.340027,1304500,631.210022\n631.380005,632.909973,625.5,1706100,625.609985\n630,635.219971,622.049988,1474200,632.590027\n628.799988,633.359985,622.650024,1575100,631.929993\n632.830017,632.830017,623.309998,1727300,628\n621,634.299988,620.5,2675400,627.26001\n647,648.169983,622.52002,3625700,623.559998\n661.27002,663.630005,641,3029100,644.280029\n660.890015,678.640015,659,3929300,662.099976\n655.210022,673,654.299988,3377200,662.299988\n659.23999,668.880005,653.01001,5860900,663.02002\n649,674.468018,645,11164900,672.929993\n565.119995,580.679993,565,4768300,579.849976\n560.130005,566.502991,556.789978,1784600,560.219971\n546.76001,565.848999,546.710022,3244100,561.099976\n532.880005,547.109985,532.400024,2206500,546.549988\n526.289978,532.559998,525.549988,1956700,530.130005\n523.119995,523.77002,520.349976,1839400,520.679993\n521.049988,522.734009,516.109985,1296700,516.830017\n523.130005,526.179993,515.179993,1597200,525.02002\n519.5,525.25,519,1280500,522.859985\n521.080017,524.650024,521.080017,1235900,523.400024\n524.72998,525.690002,518.22998,1961000,521.840027\n526.02002,526.25,520.5,2217200,520.51001\n525.01001,528.609985,520.539978,1937800,521.52002\n537.26001,537.76001,531.349976,2109100,531.690002\n538.869995,540.900024,535.22998,1335700,535.22998\n540,540,535.659973,1286600,537.840027\n539.640015,541.499023,535.25,1197500,540.47998\n539.590027,543.73999,537.530029,1250300,538.190002\n537.210022,538.25,533.01001,1893500,536.690002\n531,538.150024,530.789978,1833100,536.72998\n529.369995,530.97998,525.099976,1294200,529.26001\n528.400024,529.640015,525.559998,1071800,528.150024\n528,528.299988,524,1632700,527.200012\n531.599976,533.119995,530.159973,955800,532.330017\n538.424988,538.97998,533.02002,1217500,534.609985\n529.359985,538.359985,529.349976,1815000,536.690002\n527.559998,529.200012,523.01001,1455300,526.690002\n533.309998,534.119995,526.23999,1520600,526.830017\n536.349976,537.200012,532.52002,1388200,533.330017\n537.76001,540.590027,534.320007,1348300,536.700012\n539.909973,543.5,537.109985,1714500,540.309998\n532.929993,543,531.330017,1939000,539.179993\n536.789978,536.789978,529.76001,1904300,533.98999\n537.369995,538.630005,531.450012,2597400,532.109985\n538.01001,540.609985,536.25,1029800,539.780029\n532.799988,540.549988,531.710022,1525000,539.789978\n538.119995,539,529.880005,2406500,532.320007\n540.150024,544.190002,539.51001,1176200,540.109985\n537.950012,543.840027,535.97998,1462700,542.51001\n538.48999,542.919983,532.971985,1430800,539.27002\n533.97998,540.659973,533.039978,1966900,537.359985\n532.01001,534.820007,528.849976,2003400,532.299988\n539.179993,539.273987,530.380005,1971300,533.849976\n533.77002,539,532.409973,1403900,538.400024\n530.559998,534.322021,528.655029,1252300,529.619995\n531.599976,533.208984,525.26001,1634200,529.039978\n538.369995,541.97998,535.400024,905300,535.700012\n536.650024,541.150024,525,1527600,538.219971\n523.98999,533.460022,521.75,1546300,530.700012\n531.23999,532.380005,521.085022,1567000,524.219971\n538.210022,539.73999,530.390991,1383100,530.799988\n538.530029,544.070007,535.059998,1308000,540.780029\n538.429993,539.539978,532.099976,1768200,537.900024\n547.869995,548.590027,535.049988,2082200,537.340027\n550.469971,553.679993,546.905029,1698800,549.080017\n554.640015,556.02002,550.366028,1491000,553.679993\n563.390015,565.950012,553.200012,2398000,555.369995\n566.102584,571.14259,557.252507,4932500,565.062561\n541.002435,550.96249,540.23244,4184800,547.002472\n534.402426,541.082428,531.752397,1593500,539.367458\n537.512456,539.392429,533.677415,1844700,533.972413\n525.602352,536.092424,524.50235,1679200,535.382408\n528.662379,529.842373,521.012371,2151800,524.052386\n529.902414,535.592396,529.612373,1299800,533.802391\n528.702406,534.732432,523.22235,2318800,532.532429\n536.252409,537.572435,528.094416,2604100,530.392405\n538.412447,544.062463,537.312445,1645300,539.172466\n542.292472,542.292472,537.312445,1409400,540.012416\n541.032425,541.952428,535.492451,1557800,540.782472\n538.382457,543.852476,538.382457,1178500,541.612446\n538.08244,542.692434,536.002456,1302800,537.022404\n532.222436,538.412447,529.572407,1324400,536.767432\n540.852427,540.852427,533.849395,1716300,535.532417\n548.602502,551.142488,539.502472,1963000,542.562439\n550.00246,554.712521,546.722468,1588000,548.002468\n551.622503,553.472487,548.17249,1287500,552.032502\n553.002509,555.282504,548.132463,1897400,548.342512\n557.59255,558.90254,550.652497,1572600,555.172522\n570.50259,572.262605,558.742555,2152200,558.787539\n562.562541,574.592603,561.212525,2583200,570.192597\n560.432554,562.362529,555.832536,1643800,558.81251\n561.652513,561.722529,559.052548,2616800,560.362537\n559.392531,560.802526,556.147548,1197200,557.992512\n552.50248,559.782516,547.002472,2134500,559.502513\n551.712472,553.802493,548.002468,1805500,550.842471\n550.952514,556.852545,546.002476,1640900,554.512509\n553.502476,558.402511,544.222448,1703500,547.322503\n553.512513,556.37253,550.462462,1389600,555.512505\n555.142533,558.142521,550.682486,1820700,551.182515\n564.252539,564.852573,554.732534,1792300,555.012538\n566.862541,570.272589,563.537566,1062100,568.852557\n574.882583,576.682625,566.762536,1659100,567.687558\n575.022616,577.912621,573.412609,1389600,575.332609\n571.872619,577.112637,568.012546,1876800,573.372583\n570.452587,575.392588,566.522559,1704700,573.64261\n560.532559,572.152562,558.752531,2129600,571.342601\n554.242482,564.712541,552.902503,2410100,558.402511\n543.212476,556.142529,541.502464,2311500,555.482516\n535.90245,546.222501,535.447406,1825900,543.872489\n530.002419,536.792403,528.252381,1005000,536.092424\n536.052398,536.441404,529.412422,1457800,531.912381\n543.132484,543.75247,535.802444,1444300,538.952441\n538.042413,543.11247,538.012424,989100,542.872432\n541.402458,545.492471,537.512456,1453000,539.702422\n546.83245,550.00246,541.092465,1616800,542.842443\n543.352447,549.912491,543.132484,1900300,549.012501\n537.252405,544.822482,534.675391,1620200,542.932472\n535.302416,538.452412,533.380397,1377700,535.972405\n529.302379,537.702431,526.922378,1749800,536.942412\n528.002366,532.002411,526.022388,1267700,527.832406\n527.64237,537.202402,526.412373,1763500,531.002415\n523.792395,528.502395,522.092359,1849700,527.582391\n529.2424,532.67442,521.272361,1663600,522.762349\n528.002366,533.40243,523.262377,2038600,529.2424\n531.732383,533.002407,518.552377,2849800,528.482381\n515.862322,539.872444,515.522339,5606300,534.522445\n511.002313,511.092313,501.202274,4186300,510.6623\n522.782362,522.992349,510.002318,1683800,510.002318\n529.972369,530.702398,518.19232,1904000,518.63237\n538.532466,539.002444,529.672413,1543700,535.212448\n535.592396,542.172453,533.002407,2281700,539.952437\n521.482349,536.332401,519.702382,2676900,534.392388\n507.252283,519.282346,506.202284,2268700,518.042373\n511.002313,512.502307,506.018277,2232000,506.902294\n500.012273,508.1923,500.002267,2298200,508.082288\n505.572291,505.682303,497.762267,2715800,501.792271\n494.652237,503.232286,493.002234,2215500,500.872267\n498.842256,502.982272,492.392224,2370400,496.182251\n494.942247,495.97823,487.562205,2326700,492.552239\n504.7623,504.922285,494.792239,2071300,496.172244\n497.992268,503.48227,491.002212,3353500,502.682285\n507.002299,507.246285,499.652247,2065000,501.102268\n515.002358,516.177334,501.052266,2899900,501.962262\n523.262377,524.332389,513.062315,2059800,513.872306\n529.012399,531.272382,524.102388,1447500,524.812404\n531.252429,532.602384,525.802363,1368200,526.402397\n528.092396,531.152424,527.132366,876200,530.422394\n532.192385,535.482414,530.013375,2278500,530.332426\n528.772422,534.252417,527.312364,1040500,534.032392\n530.512424,531.761394,527.022384,705900,528.772422\n527.00237,534.56241,526.292354,2197600,530.592416\n516.082347,526.462376,516.082347,2723700,524.872383\n511.512318,517.722342,506.91328,3690200,516.352313\n512.952333,513.872306,504.70229,2926700,511.102319\n497.002248,507.002299,496.812244,2883100,504.892295\n511.562321,513.052308,489.00222,3964200,495.392242\n522.742396,523.102392,513.272333,2813400,513.802351\n523.512391,528.502395,518.662359,1994500,518.662359\n527.802416,533.922411,527.102376,1610800,528.34241\n533.082399,536.332401,525.562386,1712200,526.062353\n522.142362,534.192438,520.502366,1871300,533.37244\n527.132366,531.002415,523.792395,2329300,526.982357\n531.002415,532.892425,524.282386,2565600,525.262369\n531.1624,537.342434,528.592363,1392100,537.312445\n531.442404,535.998416,529.262414,1277900,531.322384\n533.512412,535.502427,529.802408,1526600,533.752389\n538.902438,541.412434,531.862379,2115300,533.802391\n540.622426,542.002431,536.602429,1148300,541.832471\n540.882478,541.552467,537.044437,1522900,540.372473\n539.002444,543.982471,538.60444,1789900,541.082428\n537.652428,542.702471,535.622446,1706300,539.272471\n541.612446,542.142464,536.562402,2224200,537.502419\n531.252429,535.112442,531.082408,1563300,534.832438\n535.002399,538.242425,530.082412,1392200,536.992415\n537.502419,541.942452,534.172425,1962600,535.032449\n543.582448,543.792436,534.063422,1726000,536.512399\n546.682503,546.682503,542.15244,1289500,544.402446\n549.80251,549.80251,543.482442,1339300,545.38249\n550.392507,550.462462,545.172441,1129600,547.312465\n548.492459,551.942473,546.302493,965500,550.292501\n541.462437,549.592461,541.022449,1134600,547.492463\n546.212464,546.212464,538.672437,1633700,541.012473\n545.502448,546.887472,540.972446,1333200,542.042458\n556.802542,556.802542,544.052487,2032200,545.922484\n553.002509,555.502529,549.302481,1244200,554.112486\n555.502529,557.902544,553.23251,1382200,555.222525\n559.352504,559.572529,554.752486,2035000,559.082538\n548.952461,552.802497,543.512432,1455600,550.312514\n550.00246,554.192479,546.982459,1770500,549.33247\n543.002488,548.982512,541.622422,1270900,548.902458\n537.032441,544.412483,537.032441,1185300,540.772435\n544.36248,544.882461,535.792407,1973100,539.782415\n539.322413,547.222497,535.852447,2348800,543.982471\n529.892376,539.802428,528.802412,2919200,532.712427\n525.192353,526.792383,519.112324,2336200,526.542369\n509.452317,521.762353,508.102301,2607500,520.842349\n527.252385,530.982402,508.532313,5539300,511.172305\n519.002342,529.432374,515.002358,3708500,524.512387\n531.012391,532.802396,518.302363,3719300,530.032409\n538.902438,547.192507,533.172429,2222500,537.942408\n544.992443,549.502492,533.102413,2581600,533.212394\n557.722546,565.132577,544.052487,3081900,544.492476\n571.182555,571.49261,559.062524,2524800,560.882518\n565.572566,573.882587,557.492545,1990800,572.502582\n574.402629,575.27263,563.742534,1911300,563.742534\n578.802636,581.002639,574.442595,1214600,577.352614\n573.052613,577.227637,572.502582,1141700,575.282606\n567.312567,571.912585,563.322559,1178400,570.082554\n576.012635,577.582615,567.01255,1445400,568.272597\n576.932639,579.852634,572.852602,1621700,577.36259\n571.7526,578.192625,571.172579,1282400,576.362594\n576.062638,579.2526,574.662619,1443600,577.1026\n587.552646,587.982658,574.182604,1925900,575.062581\n581.462641,589.632691,580.522624,1728100,587.992634\n586.852667,586.852667,581.002639,1471400,581.132634\n593.82271,593.951665,583.462632,1689500,587.372648\n591.502688,596.482715,589.502696,3736600,596.082692\n587.002675,589.542661,585.002622,1444500,589.272695\n580.012619,587.522656,578.777604,1692800,584.772622\n572.762572,581.502606,572.662566,1480300,579.95264\n572.94257,574.952599,568.212557,1597500,573.102616\n581.002639,581.642639,574.462608,1601600,575.622589\n580.362639,581.812661,576.262588,1221000,581.352659\n581.502606,583.502659,576.942615,977300,583.102636\n588.902662,589.002667,580.002643,1287200,581.012615\n586.602653,591.772654,586.302635,1431000,589.722659\n583.982613,586.55265,581.952632,1632300,586.082672\n580.002643,586.00268,579.222611,1458100,581.982621\n580.002643,582.992655,575.002602,1215100,577.942611\n571.852606,577.832629,571.192593,1578400,577.332601\n571.332564,572.04258,567.07155,1083700,571.602592\n569.562573,573.252625,567.102579,1292900,569.202577\n577.272622,578.492642,570.105566,1703300,571.002557\n581.262629,581.802623,576.582619,1639600,577.862619\n584.722619,585.002622,579.002647,1361400,580.202654\n583.592628,585.238621,580.642643,789000,582.562642\n583.822628,584.502655,581.14261,914800,583.372664\n585.88266,586.702658,582.572618,1036700,584.492618\n585.002622,587.342658,584.002627,978600,586.862643\n576.11258,584.512631,576.002598,1284100,582.162619\n577.862619,579.382595,570.522603,1519100,573.482626\n576.182596,577.902645,570.882599,985400,574.652582\n567.312567,575.002602,565.752564,1439200,574.782577\n564.522567,565.902572,560.882518,1542000,562.732562\n569.992585,570.492553,566.002578,1214700,567.882551\n563.562536,570.252576,560.352561,1494700,568.772565\n568.00257,569.89258,561.102543,1110900,563.362525\n561.782569,570.702601,560.002541,1334300,566.376589\n570.052564,571.982601,562.612543,1551200,565.072537\n569.042592,575.352622,564.102531,1427300,573.152619\n570.402584,575.962633,562.85252,1955200,566.072533\n580.602616,583.652668,570.002561,2102700,571.602592\n586.55265,589.502696,584.002627,1016400,587.42265\n588.752653,589.702646,583.517654,1349800,585.612633\n588.072688,592.502683,584.755668,986800,590.602697\n590.402686,591.862684,587.032665,932400,589.022681\n596.452725,599.502716,591.772654,1035100,593.352671\n593.232713,597.852683,592.502683,1233100,595.982686\n590.722655,599.652725,590.602697,1699100,594.742713\n591.752702,594.40267,585.235622,2062100,589.472645\n593.002712,596.802684,582.002635,4014100,595.082696\n579.532604,580.992601,568.61258,3016500,573.732579\n588.002671,588.402633,582.202646,1397100,582.662648\n585.742628,585.807626,576.562606,1623000,584.782659\n582.602608,585.212671,578.032641,1854000,584.872627\n571.912585,580.85263,571.422594,1621700,579.182645\n565.912548,576.592595,565.012558,1356700,571.102563\n571.582578,576.72259,569.378597,1116700,576.08259\n577.662607,579.530645,566.137592,1909500,571.092587\n583.76265,586.432631,579.592644,1064600,582.252649\n583.352651,585.01266,580.922646,714100,584.732656\n583.352651,585.442672,580.392628,1056300,582.33766\n578.32262,584.402649,576.652635,1447900,582.672624\n578.662603,579.572631,574.752588,1313700,575.282606\n577.182592,579.872648,573.802595,2236900,577.242632\n581.002639,582.45266,571.852606,1741900,576.002598\n565.262572,579.962616,565.222546,1969300,578.652627\n565.192556,572.648612,561.012513,2207100,564.622572\n555.152509,565.002582,554.252519,1536800,564.952579\n556.852545,557.582513,550.394465,4508300,556.362492\n554.242482,555.0025,548.512473,2456800,554.902495\n544.862448,553.562516,544.002484,1741700,553.372481\n544.202435,545.32245,539.33245,1444500,543.012465\n549.262515,549.622511,541.522477,1702500,544.282488\n552.262503,552.30253,545.562488,1220500,551.762475\n557.302509,557.992512,548.46247,1458400,551.352476\n558.002549,559.882522,555.022514,1100100,558.842499\n560.512546,563.602563,557.902544,1351700,560.552511\n557.152501,562.902523,556.042523,1467500,562.122552\n558.062528,558.062528,548.932509,1736700,556.332503\n546.402499,554.952498,544.452449,1689100,553.902499\n541.502464,548.612478,538.752429,1816400,544.662436\n550.99248,552.342495,542.552463,1866600,544.94244\n560.70252,560.902531,545.732448,1435000,553.932488\n560.802526,561.352557,555.912528,1771100,559.892559\n563.352549,564.002525,558.712504,1354100,560.082534\n564.57257,567.842585,561.002537,1652000,561.682564\n556.002496,566.002578,554.352525,2104100,565.952575\n547.262462,553.642508,543.702467,1932100,552.702492\n541.132431,547.602506,540.782472,1615800,545.062459\n532.902401,539.185441,531.912381,1196200,538.942465\n529.742368,536.232457,526.302392,1784700,529.772418\n519.702382,529.782394,517.58537,1277700,528.862391\n521.39238,521.802379,515.442347,1485200,520.632362\n525.702357,525.872379,517.422325,1704300,519.982324\n533.002407,533.002407,525.292358,1191700,526.652412\n530.892433,536.072411,529.512367,1653400,533.092437\n523.512391,530.192393,519.012379,1912500,529.922427\n510.752299,519.902332,504.202292,2439400,518.732375\n508.462297,517.232351,506.452298,2021300,511.002313\n515.792305,516.68232,503.302272,3224300,509.962321\n525.232379,526.812396,515.062337,1689000,515.14233\n524.822381,528.902418,521.322364,1024100,527.812392\n533.762426,534.002403,525.612389,1688500,527.932411\n527.112413,532.932391,523.882364,1905500,531.352435\n527.602405,528.002366,522.522372,1751100,526.662388\n516.902344,529.462425,516.322324,2699000,527.70241\n517.182348,518.602319,502.802274,3335500,517.152359\n522.512395,524.702361,515.422333,2100300,516.182352\n530.072374,531.652391,522.122349,1883100,525.162363\n533.792415,533.872408,526.252389,2052300,526.942391\n528.642366,537.232453,527.512375,2365300,534.812425\n536.1024,536.702435,525.602352,2566700,528.622414\n548.81249,549.502492,531.152424,6809500,536.1024\n543.002488,557.002492,540.00244,4893200,556.54249\n536.822454,538.452412,518.462348,3855000,536.442444\n538.252462,544.10249,529.56237,2575000,532.522392\n532.552381,540.00244,526.532392,3924800,530.602392\n565.002582,565.002582,539.902434,4036800,540.952433\n559.622532,565.372554,552.952506,3330800,564.142557\n542.602466,555.0025,541.612446,3151200,554.902495\n540.742445,548.482483,527.152379,4401600,538.152456\n574.652582,577.772589,543.002488,6369300,543.14246\n569.852553,587.282679,564.132581,5099100,569.742571\n599.992707,604.832763,562.192568,147100,567.002574\n558.712504,568.452595,558.712504,7900,567.162558\n566.892592,567.002574,556.932537,10800,556.972503\n561.202549,566.43259,558.672539,41200,559.992565\n568.00257,568.00257,552.922516,13100,558.462551\n"
  },
  {
    "path": "data_loader.py",
    "content": "\"\"\"Data loader.\n\nAuthor: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n----------------------------------------\nLoads Google stock dataset with MinMax normalization.\nReference: https://finance.yahoo.com/quote/GOOGL/history?p=GOOGL\n\"\"\"\n\n# Necessary Packages\nimport numpy as np\nfrom utils import MinMaxScaler\n\n\ndef data_loader(train_rate = 0.8, seq_len = 7):\n  \"\"\"Loads Google stock data.\n  \n  Args:\n    - train_rate: the ratio between training and testing sets\n    - seq_len: sequence length\n    \n  Returns:\n    - train_x: training feature\n    - train_y: training labels\n    - test_x: testing features\n    - test_y: testing labels\n  \"\"\"\n  \n  # Load data\n  ori_data = np.loadtxt('data/google.csv', delimiter=',', skiprows = 1)\n  # Reverse the time order\n  reverse_data = ori_data[::-1]\n  # Normalization\n  norm_data = MinMaxScaler(reverse_data)\n    \n  # Build dataset\n  data_x = []\n  data_y = []\n  \n  for i in range(0, len(norm_data[:,0]) - seq_len):\n    # Previous seq_len data as features\n    temp_x = norm_data[i:i + seq_len,:]\n    # Values at next time point as labels\n    temp_y = norm_data[i + seq_len, [-1]]\n    data_x = data_x + [temp_x]\n    data_y = data_y + [temp_y]\n    \n  data_x = np.asarray(data_x)\n  data_y = np.asarray(data_y)\n            \n  # Train / test Division   \n  idx = np.random.permutation(len(data_x))\n  train_idx = idx[:int(train_rate * len(data_x))]\n  test_idx = idx[int(train_rate * len(data_x)):]\n        \n  train_x, test_x = data_x[train_idx, :, :], data_x[test_idx, :, :]\n  train_y, test_y = data_y[train_idx, :], data_y[test_idx, :]\n    \n  return train_x, train_y, test_x, test_y\n"
  },
  {
    "path": "main_time_series_prediction.py",
    "content": "\"\"\"Time-series prediction main function\n\nAuthor: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n------------------------------------\n(1) Load data\n(2) Train model (RNN, GRU, LSTM, Attention)\n(3) Evaluate the trained model\n\"\"\"\n\n# Necessary packages\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport argparse\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom data_loader import data_loader\nfrom basic_rnn_lstm_gru import GeneralRNN\nfrom basic_attention import Attention\nfrom utils import performance\n\n\ndef main (args):  \n  \"\"\"Time-series prediction main function.\n  \n  Args:\n    - train_rate: training data ratio\n    - seq_len: sequence length\n    - task: classification or regression\n    - model_type: rnn, lstm, gru, or attention\n    - h_dim: hidden state dimensions\n    - n_layer: number of layers\n    - batch_size: the number of samples in each mini-batch\n    - epoch: the number of iterations\n    - learning_rate: learning rates\n    - metric_name: mse or mae\n  \"\"\"\n  # Load data\n  train_x, train_y, test_x, test_y = data_loader(args.train_rate, \n                                                 args.seq_len)\n  \n  # Model traininig / testing\n  model_parameters = {'task': args.task,\n                      'model_type': args.model_type,\n                      'h_dim': args.h_dim,\n                      'n_layer': args.n_layer,\n                      'batch_size': args.batch_size,\n                      'epoch': args.epoch,\n                      'learning_rate': args.learning_rate}\n  \n  if args.model_type in ['rnn','lstm','gru']:\n    general_rnn = GeneralRNN(model_parameters)    \n    general_rnn.fit(train_x, train_y)\n    test_y_hat = general_rnn.predict(test_x)\n  elif args.model_type == 'attention':\n    basic_attention = Attention(model_parameters)    \n    basic_attention.fit(train_x, train_y)\n    test_y_hat = basic_attention.predict(test_x)\n  \n  # Evaluation\n  result = performance(test_y, test_y_hat, args.metric_name)\n  print('Performance (' + args.metric_name + '): ' + str(result))\n  \n    \n##  \nif __name__ == '__main__':\n  \n  # Inputs for the main function\n  parser = argparse.ArgumentParser()\n  parser.add_argument(\n      '--train_rate',\n      help='training data ratio',\n      default=0.8,\n      type=str)\n  parser.add_argument(\n      '--seq_len',\n      help='sequence length',\n      default=7,\n      type=int)\n  parser.add_argument(\n      '--model_type',\n      choices=['rnn','gru','lstm','attention'],\n      default='attention',\n      type=str)\n  parser.add_argument(\n      '--h_dim',\n      default=10,\n      type=int)\n  parser.add_argument(\n      '--n_layer',\n      default=3,\n      type=int)\n  parser.add_argument(\n      '--batch_size',\n      default=32,\n      type=int)\n  parser.add_argument(\n      '--epoch',\n      default=100,\n      type=int)\n  parser.add_argument(\n      '--learning_rate',\n      default=0.01,\n      type=float)\n  parser.add_argument(\n      '--task',\n      choices=['classification','regression'],\n      default='regression',\n      type=str)\n  parser.add_argument(\n      '--metric_name',\n      choices=['mse','mae'],\n      default='mae',\n      type=str)\n  \n  args = parser.parse_args() \n  \n  # Call main function  \n  main(args)"
  },
  {
    "path": "utils.py",
    "content": "\"\"\"Utility functions for time-series prediction.\n\nAuthor: Jinsung Yoon\nContact: jsyoon0823@gmail.com\n------------------------------------\n(1) MinMaxScaler: MinMax normalizer\n(2) performance: performance evaluator\n(3) binary_cross_entropy_loss: loss for RNN on classification\n(4) mse_loss: loss for RNN on regression\n(5) rnn_sequential: Architecture\n\"\"\"\n\n# Necessary packages\nimport numpy as np\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nimport tensorflow as tf\nfrom tensorflow.keras import layers\n\n\ndef MinMaxScaler(data):    \n  \"\"\"Normalizer (MinMax criteria).\n  \n  Args:\n    - data: original data\n    \n  Returns:\n    - norm_data: normalized data\n  \"\"\"    \n  numerator = data - np.min(data, 0)\n  denominator = np.max(data, 0) - np.min(data, 0)\n  norm_data = numerator / (denominator + 1e-8)\n  return norm_data\n\n\ndef performance(test_y, test_y_hat, metric_name):\n  \"\"\"Evaluate predictive model performance.\n  \n  Args:\n    - test_y: original testing labels\n    - test_y_hat: prediction on testing data\n    - metric_name: 'mse' or 'mae'\n    \n  Returns:\n    - score: performance of the predictive model\n  \"\"\"  \n  assert metric_name in ['mse', 'mae']\n  \n  if metric_name == 'mse':\n    score = mean_squared_error(test_y, test_y_hat)\n  elif metric_name == 'mae':\n    score = mean_absolute_error(test_y, test_y_hat)\n    \n  score = np.round(score, 4)\n    \n  return score\n\n\ndef binary_cross_entropy_loss (y_true, y_pred):\n  \"\"\"User defined cross entropy loss.\n  \n  Args:\n    - y_true: true labels\n    - y_pred: predictions\n    \n  Returns:\n    - loss: computed loss\n  \"\"\"\n  # Exclude masked labels\n  idx = tf.cast((y_true >= 0), float)\n  # Cross entropy loss excluding masked labels\n  loss = -(idx * y_true * tf.math.log(y_pred) + \\\n           idx * (1-y_true) * tf.math.log(1-y_pred))\n  return loss\n\n\ndef mse_loss (y_true, y_pred):\n  \"\"\"User defined mean squared loss.\n  \n  Args:\n    - y_true: true labels\n    - y_pred: predictions\n    \n  Returns:\n    - loss: computed loss\n  \"\"\"\n  # Exclude masked labels\n  idx = tf.cast((y_true >= 0), float)\n  # Mean squared loss excluding masked labels\n  loss = idx * tf.pow(y_true - y_pred, 2)\n  return loss\n\n\ndef rnn_sequential (model, model_name, h_dim, return_seq):\n  \"\"\"Add one rnn layer in sequential model.\n  \n  Args:\n    - model: sequential rnn model\n    - model_name: rnn, lstm, or gru\n    - h_dim: hidden state dimensions\n    - return_seq: True or False\n    \n  Returns:\n    - model: sequential rnn model\n  \"\"\"\n  \n  if model_name == 'rnn':\n    model.add(layers.SimpleRNN(h_dim, return_sequences=return_seq))\n  elif model_name == 'lstm':\n    model.add(layers.LSTM(h_dim, return_sequences=return_seq))\n  elif model_name == 'gru':\n    model.add(layers.GRU(h_dim, return_sequences=return_seq))\n    \n  return model"
  }
]