Full Code of jsyoon0823/Time-series-prediction for AI

Repository: jsyoon0823/Time-series-prediction
Branch: master
Commit: 28970208aa3b
Files: 7
Total size: 56.1 KB

Directory structure:
gitextract_b2oaa3yp/

├── README.md
├── basic_attention.py
├── basic_rnn_lstm_gru.py
├── data/
│   └── google.csv
├── data_loader.py
├── main_time_series_prediction.py
└── utils.py

================================================
FILE CONTENTS
================================================

================================================
FILE: README.md
================================================
# Codebase for "Time-series prediction" with RNN, GRU, LSTM and Attention

Authors: Jinsung Yoon
Contact: jsyoon0823@gmail.com

This directory contains implementations of basic time-series prediction
using RNN, GRU, LSTM or Attention methods.
To run the pipeline, simply run python3 -m main_time_series_prediction.py.

Note: We recommend to do MinMax normalization on both input and output.

## Stages of time-series prediction framework:

-   Load dataset (Google stocks data)
-   Train model:
    (1) RNN based: Simple RNN, GRU, LSTM
    (2) Attention based
-   Evaluate the performance: MAE or MSE metrics

### Command inputs:

-   train_rate: training data ratio
-   seq_len: sequence length
-   task: classification or regression
-   model_type: rnn, lstm, gru, or attention
-   h_dim: hidden state dimensions
-   n_layer: number of layers
-   batch_size: the number of samples in each mini-batch
-   epoch: the number of iterations
-   learning_rate: learning rates
-   metric_name: mse or mae

### Example command

```shell
$ python3 main_time_series_prediction.py 
--train_rate 0.8 --seq_len 7 --task regression --model_type lstm
--h_dim 10 --n_layer 3 --batch_size 32 --epoch 100 --learning_rate 0.01
--metric_name mae
```

### Outputs

-   MAE or MSE performance of trained model


================================================
FILE: basic_attention.py
================================================
"""Basic Attention core functions for time-series prediction.

Author: Jinsung Yoon
Contact: jsyoon0823@gmail.com
"""

# Necessary packages
import tensorflow as tf
import numpy as np
import os
import shutil


class Attention():
  """Attention class.
  
  Attributes:
    - model_parameters:
      - task: classificiation or regression
      - h_dim: hidden state dimensions
      - batch_size: the number of samples in each mini-batch
      - epoch: the number of iterations
      - learning_rate: learning rate of training
  """
  def __init__(self, model_parameters):

    tf.compat.v1.reset_default_graph()
    self.task = model_parameters['task']
    self.h_dim = model_parameters['h_dim']
    self.batch_size = model_parameters['batch_size']
    self.epoch = model_parameters['epoch']
    self.learning_rate = model_parameters['learning_rate']
    
    self.save_file_directory = 'tmp/attention/'
    
    
  def process_batch_input_for_RNN(self, batch_input):
    """Function to convert batch input data to use scan ops of tensorflow.
    
    Args:
      - batch_input: original batch input
    
    Returns:
      - x: batch_input for RNN 
    """
    batch_input_ = tf.transpose(batch_input, perm=[2, 0, 1])
    x = tf.transpose(batch_input_)
    return x


  def sample_X(self, m, n):
    """Sample from the real data (Mini-batch index sampling).
    """
    return np.random.permutation(m)[:n]  
  
  
  def fit(self, x, y):
    """Train the model.
    
    Args:
      - x: training feature
      - y: training label
    """
        
    # Basic parameters
    no, seq_len, x_dim = x.shape
    y_dim = len(y[0, :])
    
    # Weights for GRU
    Wr = tf.Variable(tf.zeros([x_dim, self.h_dim]))
    Ur = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))
    br = tf.Variable(tf.zeros([self.h_dim]))
        
    Wu = tf.Variable(tf.zeros([x_dim, self.h_dim]))
    Uu = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))
    bu = tf.Variable(tf.zeros([self.h_dim]))
        
    Wh = tf.Variable(tf.zeros([x_dim, self.h_dim]))
    Uh = tf.Variable(tf.zeros([self.h_dim, self.h_dim]))
    bh = tf.Variable(tf.zeros([self.h_dim]))
                
    # Weights for attention mechanism 
    Wa1 = tf.Variable(tf.random.truncated_normal([self.h_dim + x_dim, 
                                                  self.h_dim], 
                                                 mean=0, stddev=.01))
    Wa2 = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], 
                                                 mean=0, stddev=.01))
    ba1 = tf.Variable(tf.random.truncated_normal([self.h_dim], 
                                                 mean=0, stddev=.01))
    ba2 = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01))
            
    # Weights for output layers
    Wo = tf.Variable(tf.random.truncated_normal([self.h_dim, y_dim], 
                                         mean=0, stddev=.01))
    bo = tf.Variable(tf.random.truncated_normal([y_dim], mean=0, stddev=.01))
    
    # Target
    Y = tf.compat.v1.placeholder(tf.float32, [None,1])    
    # Input vector with shape[batch, seq, embeddings]
    _inputs = tf.compat.v1.placeholder(tf.float32, shape=[None, None, x_dim], 
                                       name='inputs')
    
    # Processing inputs to work with scan function
    processed_input = self.process_batch_input_for_RNN(_inputs)
            
    # Initial Hidden States
    initial_hidden = _inputs[:, 0, :]
    initial_hidden = tf.matmul(initial_hidden, tf.zeros([x_dim, self.h_dim]))
        
 
    def GRU(previous_hidden_state, x):
      """Function for Forward GRU cell.
      
      Args:
        - previous_hidden_state
        - x: current input
        
      Returns:
        - current_hidden_state
      """
      # R Gate
      r = tf.sigmoid(tf.matmul(x, Wr) + \
                     tf.matmul(previous_hidden_state, Ur) + br)
      # U Gate
      u = tf.sigmoid(tf.matmul(x, Wu) + \
                     tf.matmul(previous_hidden_state, Uu) + bu)
      # Final Memory cell
      c = tf.tanh(tf.matmul(x, Wh) + \
                  tf.matmul( tf.multiply(r, previous_hidden_state), Uh) + bh)
      # Current Hidden state
      current_hidden_state = tf.multiply( (1 - u), previous_hidden_state ) + \
                             tf.multiply( u, c )
      return current_hidden_state
        
    
    def get_states():
      """Function to get the hidden and memory cells after forward pass.
      
      Returns:
        - all_hidden_states
      """
      # Getting all hidden state through time
      all_hidden_states = tf.scan(GRU, processed_input, 
                                  initializer=initial_hidden, name='states')
      return all_hidden_states
              
        
    def get_attention(hidden_state):
      """Function to get attention with the last input.
      
      Args:
        - hidden_states
        
      Returns:
        - e_values
      """
      inputs = tf.concat((hidden_state, processed_input[-1]), axis = 1)
      hidden_values = tf.nn.tanh(tf.matmul(inputs, Wa1) + ba1)
      e_values = (tf.matmul(hidden_values, Wa2) + ba2)
      return e_values
        

    def get_outputs():
      """Function for getting output and attention coefficient.
      
      Returns:
        - output: final outputs
        - a_values: attention values
      """
      all_hidden_states = get_states()
      all_attention = tf.map_fn(get_attention, all_hidden_states)
      a_values = tf.nn.softmax(all_attention, axis = 0)
      final_hidden_state = tf.einsum('ijk,ijl->jkl', a_values, 
                                     all_hidden_states)
      output = tf.nn.sigmoid(tf.matmul(final_hidden_state[:,0,:], Wo) + bo, 
                             name='outputs')
      return output, a_values   
                
    # Getting all outputs from rnn
    outputs, attention_values = get_outputs()
    
    # reshape out for sequence_loss
    if self.task == 'classification':
      loss = tf.reduce_mean(Y * tf.log(outputs + 1e-8) + \
                            (1-Y) * tf.log(1-outputs + 1e-8))
    elif self.task == 'regression':
      loss = tf.sqrt(tf.reduce_mean(tf.square(outputs - Y)))
    
    # Optimization
    optimizer = tf.compat.v1.train.AdamOptimizer(self.learning_rate)
    train = optimizer.minimize(loss)

    # Sessions
    sess = tf.compat.v1.Session()
    sess.run(tf.compat.v1.global_variables_initializer())
        
    # Training
    iteration_per_epoch = int(no/self.batch_size)
    iterations = int((self.epoch * no) / self.batch_size)
    
    for i in range(iterations):
      
      idx = self.sample_X(no, self.batch_size)
      Input = x[idx,:,:]            
      _, step_loss = sess.run([train, loss], 
                              feed_dict={Y: y[idx], _inputs: Input})
                
      # Print intermediate results
      if i % iteration_per_epoch == iteration_per_epoch-1:
        print('Epoch: ' + str(int(i/iteration_per_epoch)) + 
              ', Loss: ' + str(np.round(step_loss, 4)))
        
    # Reset the directory for saving
    if not os.path.exists(self.save_file_directory):
      os.makedirs(self.save_file_directory)
    else:
      shutil.rmtree(self.save_file_directory)
  
    # Save model
    inputs = {'inputs': _inputs}
    outputs = {'outputs': outputs}
    tf.compat.v1.saved_model.simple_save(sess, self.save_file_directory, 
                                         inputs, outputs)    
        
       
  def predict(self, test_x):
    """Prediction with trained model.
    
    Args:
      - test_x: testing features
      
    Returns:
      - test_y_hat: predictions on testing set
    """
    
    graph = tf.Graph()
    with graph.as_default():
      with tf.compat.v1.Session() as sess:
        tf.compat.v1.saved_model.loader.load(sess, [tf.saved_model.SERVING], 
                                             self.save_file_directory)
        x = graph.get_tensor_by_name('inputs:0')
        outputs = graph.get_tensor_by_name('outputs:0')
    
        test_y_hat = sess.run(outputs, feed_dict={x: test_x})
        
    return test_y_hat
    

================================================
FILE: basic_rnn_lstm_gru.py
================================================
"""General RNN core functions for time-series prediction.

Author: Jinsung Yoon
Contact: jsyoon0823@gmail.com
"""

# Necessary packages
import os
import tensorflow as tf
import numpy as np
from datetime import datetime
from tensorflow.keras import layers
from keras.callbacks import ModelCheckpoint
from utils import binary_cross_entropy_loss, mse_loss, rnn_sequential


class GeneralRNN():
  """RNN predictive model to time-series.
  
  Attributes:
    - model_parameters:
      - task: classification or regression
      - model_type: 'rnn', 'lstm', or 'gru'
      - h_dim: hidden dimensions
      - n_layer: the number of layers
      - batch_size: the number of samples in each batch
      - epoch: the number of iteration epochs
      - learning_rate: the learning rate of model training
  """

  def __init__(self, model_parameters):

    self.task = model_parameters['task']
    self.model_type = model_parameters['model_type']
    self.h_dim = model_parameters['h_dim']
    self.n_layer = model_parameters['n_layer']
    self.batch_size = model_parameters['batch_size']
    self.epoch = model_parameters['epoch']
    self.learning_rate = model_parameters['learning_rate']
    
    assert self.model_type in ['rnn', 'lstm', 'gru']

    # Predictor model define
    self.predictor_model = None

    # Set path for model saving
    model_path = 'tmp'
    if not os.path.exists(model_path):
      os.makedirs(model_path)
    self.save_file_name = '{}'.format(model_path) + \
                          datetime.now().strftime('%H%M%S') + '.hdf5'
  

  def _build_model(self, x, y):
    """Construct the model using feature and label statistics.
    
    Args:
      - x: features
      - y: labels
      
    Returns:
      - model: predictor model
    """    
    # Parameters
    h_dim = self.h_dim
    n_layer = self.n_layer
    dim = len(x[0, 0, :])
    max_seq_len = len(x[0, :, 0])

    model = tf.keras.Sequential()
    model.add(layers.Masking(mask_value=0., input_shape=(max_seq_len, dim)))

    for _ in range(n_layer - 1):
      model = rnn_sequential(model, self.model_type, h_dim, return_seq=True)

    model = rnn_sequential(model, self.model_type, h_dim, 
                           return_seq=False)
    adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate, 
                                    beta_1=0.9, beta_2=0.999, amsgrad=False)

    if self.task == 'classification':
      model.add(layers.Dense(y.shape[-1], activation='sigmoid'))
      model.compile(loss=binary_cross_entropy_loss, optimizer=adam)
      
    elif self.task == 'regression':
      model.add(layers.Dense(y.shape[-1], activation='linear'))
      model.compile(loss=mse_loss, optimizer=adam, metrics=['mse'])

    return model
  

  def fit(self, x, y):
    """Fit the predictor model.
    
    Args:
      - x: training features
      - y: training labels
      
    Returns:
      - self.predictor_model: trained predictor model
    """
    idx = np.random.permutation(len(x))
    train_idx = idx[:int(len(idx)*0.8)]
    valid_idx = idx[int(len(idx)*0.8):]
    
    train_x, train_y = x[train_idx], y[train_idx]
    valid_x, valid_y = x[valid_idx], y[valid_idx]
    
    self.predictor_model = self._build_model(train_x, train_y)

    # Callback for the best model saving
    save_best = ModelCheckpoint(self.save_file_name, monitor='val_loss',
                                mode='min', verbose=False,
                                save_best_only=True)

    # Train the model
    self.predictor_model.fit(train_x, train_y, 
                             batch_size=self.batch_size, epochs=self.epoch, 
                             validation_data=(valid_x, valid_y), 
                             callbacks=[save_best], verbose=True)

    self.predictor_model.load_weights(self.save_file_name)
    os.remove(self.save_file_name)

    return self.predictor_model
  
  
  def predict(self, test_x):
    """Return the temporal and feature importance.
    
    Args:
      - test_x: testing features
      
    Returns:
      - test_y_hat: predictions on testing set
    """
    test_y_hat = self.predictor_model.predict(test_x)
    return test_y_hat


================================================
FILE: data/google.csv
================================================
Open,High,Low,Volume,Close
828.659973,833.450012,828.349976,1247700,831.659973
823.02002,828.070007,821.655029,1597800,828.070007
819.929993,824.400024,818.97998,1281700,824.159973
819.359985,823,818.469971,1304000,818.97998
819,823,816,1053600,820.450012
816,820.958984,815.48999,1198100,819.23999
811.700012,815.25,809.780029,1129100,813.669983
809.51001,810.659973,804.539978,989700,809.559998
807,811.840027,803.190002,1155300,808.380005
803.98999,810.5,801.780029,1235200,806.969971
799.700012,801.669983,795.25,1174200,801.340027
802.98999,806,800.369995,1460400,801.48999
793.799988,802.700012,792,1525800,798.530029
799.679993,801.190002,791.190002,2023300,795.695007
796.859985,801.25,790.52002,2143500,796.789978
814.659973,815.840027,799.799988,3228900,802.320007
834.710022,841.950012,820.440002,2951800,823.309998
837.809998,838,827.01001,2734400,832.150024
829.619995,835.77002,825.059998,1494500,835.669983
822.299988,825.900024,817.820984,1461000,823.869995
807.25,820.869995,803.73999,1901600,819.309998
806.909973,806.909973,801.690002,1645000,805.02002
805.119995,809.47998,801.799988,912000,802.174988
805.809998,806.205017,800.98999,1293300,806.070007
807.080017,807.140015,800.369995,1355800,804.609985
807.47998,811.223999,806.690002,1090100,807.880005
807.140015,807.390015,799.169983,1351000,806.359985
805,808.150024,801.369995,1057900,807.909973
807.859985,809.130005,803.51001,1174300,804.789978
806.400024,809.966003,802.830017,1272400,806.650024
795.26001,807.900024,792.203979,1620500,806.150024
786.080017,794.47998,785.02002,1315400,794.02002
788.359985,791.340027,783.159973,1065400,786.900024
778.809998,789.630005,775.799988,1643100,786.140015
782.75,782.780029,770.409973,1760200,771.820007
783.330017,785.929993,778.919983,742200,782.789978
793.700012,794.22998,783.200012,1132700,785.049988
790.679993,797.859985,787.656982,789100,791.549988
790.900024,792.73999,787.280029,623400,789.909973
792.359985,793.320007,788.580017,969100,791.26001
795.840027,796.676025,787.099976,1208700,794.559998
796.76001,798.650024,793.27002,925100,796.419983
790.219971,797.659973,786.27002,1225900,794.200012
800.400024,800.856018,790.289978,2428300,790.799988
797.340027,803,792.919983,1623700,797.849976
797.400024,804,794.01001,1664500,797.070007
793.900024,804.380005,793.340027,2119800,796.099976
785.039978,791.25,784.35498,2097100,789.27002
780,789.429993,779.020996,1780500,789.289978
772.47998,778.179993,767.22998,1448200,776.419983
761,771.359985,755.799988,1750800,771.190002
764.72998,768.830017,757.340027,1632800,759.109985
757.710022,763.900024,752.900024,1382900,762.52002
744.590027,754,743.099976,1448700,750.5
757.440002,759.849976,737.025024,2996900,747.919983
770.070007,772.98999,754.830017,2365800,758.039978
771.530029,778.5,768.23999,1604500,770.840027
760,779.530029,759.799988,2172200,768.23999
764.26001,765,760.52002,586000,761.679993
767.72998,768.28302,755.25,1477400,760.98999
772.630005,776.960022,767,1591800,768.27002
762.609985,769.700012,760.599976,1323000,769.200012
771.369995,775,760,1526200,760.539978
766.919983,772.700012,764.22998,1287000,771.22998
755.200012,766.359985,750.51001,1465200,764.47998
746.969971,764.416016,746.969971,2341200,758.48999
755.599976,757.849976,727.539978,3631700,736.080017
756.539978,760.780029,750.380005,2394000,754.02002
791.169983,791.169983,752.179993,4740100,762.559998
779.940002,791.22699,771.669983,2602700,785.309998
783.400024,795.632996,780.190002,1350800,790.51001
774.5,785.190002,772.549988,1585100,782.52002
750.659973,770.359985,750.560974,2134800,762.02002
767.25,769.950012,759.030029,1943200,762.130005
778.200012,781.650024,763.450012,1872400,768.700012
782.890015,789.48999,775.539978,2406400,783.609985
795.469971,796.859985,784,2427300,784.539978
808.349976,815.48999,793.590027,4269900,795.369995
801,803.48999,791.5,2749200,795.349976
806.340027,806.97998,796.320007,1647700,799.070007
816.679993,816.679993,805.140015,1576400,807.669983
804.900024,815.179993,804.820007,1697500,813.109985
795,799.5,794,1266200,799.369995
803.299988,803.969971,796.030029,1757500,796.969971
798.23999,804.599976,798.030029,1766800,801.5
787.849976,801.609985,785.565002,2056900,795.26001
779.799988,785.849976,777.5,1093000,779.960022
781.650024,783.950012,776,852500,778.530029
781.219971,781.219971,773,1365300,778.190002
783.76001,788.130005,782.059998,937400,786.140015
786.659973,792.280029,780.580017,1372500,783.070007
777.710022,789.380005,775.869995,1174900,785.940002
779.659973,779.659973,770.75,933200,775.080017
779,780.47998,775.539978,1070700,776.859985
779.309998,782.070007,775.650024,1461200,776.469971
776.030029,778.710022,772.890015,1201400,776.429993
774.25,776.065002,769.5,1278800,772.559998
776.330017,780.940002,774.090027,1585300,777.289978
781.440002,785.799988,774.231995,1314700,775.01001
777.849976,781.809998,774.969971,1109800,781.559998
775.5,785.98999,774.307983,1153200,783.01001
782.73999,782.73999,773.070007,1533200,774.210022
786.590027,788.929993,784.150024,1411900,786.900024
780,789.849976,778.440002,1486200,787.210022
772.659973,777.159973,768.301025,1167800,776.219971
769,773.330017,768.530029,978600,771.409973
772.419983,774,764.440979,1172800,765.700012
769.75,769.75,764.659973,2049300,768.880005
762.890015,773.799988,759.960022,1305100,771.76001
759.609985,767.679993,759.109985,1087400,762.48999
764.47998,766.219971,755.799988,1395000,759.690002
755.130005,770.289978,754,1311000,769.02002
770.099976,773.244995,759.659973,1885500,759.659973
778.590027,780.349976,773.580017,1270300,775.320007
780,782.72998,776.200012,893700,780.349976
773.450012,782,771,1442800,780.080017
773.01001,773.919983,768.409973,1072700,771.460022
769.25,771.02002,764.299988,925100,768.780029
767.01001,769.090027,765.380005,1248600,767.049988
769.330017,774.466003,766.840027,1130000,769.090027
768.73999,774.98999,766.61499,847600,772.150024
769,776.080017,765.849976,1166700,769.539978
767,771.890015,763.184998,926900,769.409973
770.580017,774.5,767.070007,1072000,769.640015
775.47998,776.440002,771.784973,928200,772.080017
773.27002,774.539978,770.049988,951400,772.150024
775,777.099976,773.130005,861500,775.419983
780.01001,782.859985,777,719400,777.5
777.320007,780.809998,773.530029,924200,779.909973
780.299988,780.97998,773.44397,1028000,777.140015
783.75,787.48999,780.109985,938200,782.440002
781.5,783.39502,780.400024,740500,783.219971
785,789.75,782.969971,975100,784.849976
783.75,786.812012,782.778015,786400,784.679993
781.099976,788.940002,780.570007,1318900,784.26001
782,782.630005,778.091003,1107900,781.76001
773.780029,783.039978,772.340027,1801200,782.219971
772.219971,774.070007,768.794983,1140300,771.609985
767.179993,773.210022,766.820007,1287400,773.179993
768.690002,775.840027,767.849976,1784500,771.070007
761.090027,780.429993,761.090027,2700500,772.880005
772.710022,778.549988,766.77002,3841500,768.789978
747.039978,748.650024,739.299988,3530200,745.909973
738.280029,744.460022,737,1512500,741.77002
739.039978,741.690002,734.27002,1186700,738.419983
740.669983,742.609985,737.5,1032400,739.77002
741.859985,743.23999,736.559998,1259800,742.73999
740.359985,741.690002,735.830994,1026300,738.630005
737.330017,742.130005,737.099976,1289700,741.190002
729.890015,736.98999,729,1227500,736.960022
722.710022,736.130005,721.190002,1295500,733.780029
725.72998,725.73999,719.054993,1279300,719.849976
721.580017,722.210022,718.030029,950200,720.950012
723.619995,724,716.849976,935900,716.97998
719.119995,722.940002,715.909973,1336900,720.640015
708.049988,716.51001,707.23999,1111800,715.090027
699.5,705.710022,696.434998,1575200,705.630005
698.080017,698.200012,688.215027,1304200,695.359985
689.97998,701.679993,689.090027,1411900,697.77002
696.059998,696.940002,688.880005,1462600,694.950012
692.200012,700.650024,692.130005,1344700,699.210022
685.469971,692.320007,683.650024,1597700,692.099976
683,687.429016,681.409973,1932600,684.109985
678.969971,680.330017,673,2173800,680.039978
671,672.299988,663.283997,2629000,668.26001
675.169983,689.400024,673.450012,4449000,675.219971
697.450012,701.950012,687,2171400,701.869995
699.059998,700.859985,693.08197,1184300,697.460022
698.400024,702.77002,692.01001,1465600,695.940002
698.77002,702.47998,693.409973,2082500,693.710022
708.650024,708.820007,688.452026,3402400,691.719971
714.909973,716.650024,703.26001,1982500,710.359985
719,722.97998,717.309998,1214500,718.919983
716.47998,722.469971,713.119995,1306100,718.27002
716.51001,725.440002,716.51001,1258900,718.359985
719.469971,725.890015,716.429993,1216400,719.409973
722.869995,729.539978,722.335999,988900,728.580017
723.960022,728.570007,720.580017,1583700,728.280029
719.840027,721.97998,716.549988,1336200,716.650024
724.909973,724.909973,714.609985,1565300,716.549988
729.27002,729.48999,720.559998,1226300,722.340027
732.5,733.02002,724.169983,1341800,730.400024
734.530029,737.210022,730.659973,1253600,734.150024
731.73999,739.72998,731.26001,2129500,735.719971
724.01001,733.935974,724,1974000,732.659973
722.869995,728.330017,720.280029,1576300,724.119995
720.76001,727.51001,719.705017,1629200,725.27002
706.859985,720.969971,706.859985,1929500,720.090027
706.530029,711.478027,704.179993,1330700,704.23999
701.619995,714.580017,700.52002,1828400,709.73999
702.359985,706,696.799988,1670200,700.320007
703.669983,711.599976,700.630005,1766800,706.630005
715.98999,721.52002,704.109985,2001200,706.22998
709.130005,718.47998,705.650024,1317100,716.48999
711.929993,716.661987,709.26001,1314500,710.830017
717.059998,719.25,709,1360700,713.309998
723.409973,724.47998,712.799988,1692100,715.289978
716.75,723.5,715.719971,1569600,723.179993
712,718.710022,710,1510300,712.900024
698.380005,711.859985,698.106995,1826100,711.119995
697.700012,702.320007,695.719971,1683500,701.429993
690.48999,699.75,689.01001,1693500,695.700012
696.869995,697.840027,692,1543800,692.359985
697.630005,700.640015,691,1645300,698.210022
690.700012,697.619995,689,2487700,693.01001
708.26001,714.169983,689.549988,2867300,691.02002
707.289978,708.97998,692.36499,3098600,705.840027
725.419983,725.765991,703.026001,2744600,708.140015
716.099976,723.929993,715.590027,1959200,723.150024
726.299988,736.119995,713.609985,5951900,718.77002
755.380005,760.450012,749.549988,3060500,759.140015
758,758.132019,750.01001,1529200,752.669983
769.51001,769.900024,749.330017,2030500,753.929993
760.460022,768.049988,757.299988,1556000,766.609985
753.97998,761,752.69397,1809300,759
754.01001,757.309998,752.705017,1135300,753.200012
749.159973,754.380005,744.260986,1707100,751.719971
738,743.830017,731.01001,1353000,743.090027
743.02002,745,736.049988,1220100,736.099976
743.969971,745.450012,735.549988,1290800,739.150024
745.369995,747,736.280029,1453200,740.280029
735.77002,746.23999,735.559998,1053700,745.690002
738,742.799988,735.369995,1132300,737.799988
750.059998,752.799988,742.429993,1134200,745.289978
738.599976,750.340027,737,1576700,749.909973
749.25,750.849976,740.940002,1718800,744.950012
750.099976,757.880005,748.73999,1782400,750.530029
734.590027,747.25,728.76001,1903800,744.77002
736.789978,738.98999,732.5,1301300,733.530029
732.01001,737.747009,731,1594900,735.299988
742.359985,745.719971,736.150024,1432100,738.059998
737.460022,745,737.460022,1269700,740.75
736.5,742.5,733.515991,1836500,742.090027
741.859985,742,731.830017,2980700,737.599976
736.450012,743.070007,736,1860800,737.780029
726.369995,737.469971,724.51001,1624400,736.090027
726.919983,732.289978,724.77002,1721000,728.330017
726.809998,735.5,725.150024,1718300,730.48999
720,726.919983,717.125,1970800,726.820007
708.119995,716.440002,703.359985,2833500,712.820007
698.469971,705.679993,694,1421500,705.23999
688.590027,703.789978,685.340027,2076300,693.969971
706.900024,708.091003,686.900024,2985100,695.159973
714.98999,716.48999,706.02002,1972100,710.890015
718.679993,719.450012,706.02002,1958000,712.419983
719,720,712,1629000,718.849976
703.619995,718.809998,699.77002,2151400,718.809998
700.320007,710.890015,697.679993,2481100,697.77002
708.580017,713.429993,700.859985,2243500,705.070007
700.01001,705.97998,690.585022,1642200,705.75
688.919983,700,680.780029,1963600,699.559998
701.450012,708.400024,693.580017,2009300,695.849976
707.450012,713.23999,702.51001,1949800,706.460022
695.030029,703.080994,694.049988,1589300,700.909973
710,712.349976,696.030029,1883200,697.349976
698.090027,709.75,691.380005,2492600,708.400024
692.97998,698,685.049988,2520000,691
690.26001,693.75,678.599976,2141400,682.400024
675,689.349976,668.867981,3024000,683.109985
686.859985,701.309998,682.130005,2638000,684.119995
672.320007,699.900024,668.77002,3608900,678.109985
667.849976,684.030029,663.059998,4247400,682.73999
703.869995,703.98999,680.150024,5105700,683.570007
722.809998,727,701.859985,5168700,708.01001
770.219971,774.5,720.5,6171000,726.950012
784.5,789.869995,764.650024,6348100,764.650024
750.460022,757.859985,743.27002,5139200,752
731.530029,744.98999,726.799988,3474300,742.950012
722.219971,733.690002,712.349976,2676400,730.960022
713.669983,718.234985,694.390015,2194200,699.98999
713.849976,718.280029,706.47998,1331700,713.039978
723.580017,729.679993,710.01001,1711700,711.669983
723.599976,728.130005,720.120972,2011800,725.25
702.179993,719.190002,694.460022,2412200,706.590027
688.609985,706.849976,673.26001,3445000,698.450012
703.299988,709.97998,693.409973,2268100,701.789978
692.289978,706.73999,685.369995,3592400,694.450012
705.380005,721.924988,689.099976,2225800,714.719971
730.849976,734.73999,698.609985,2501700,700.559998
721.679993,728.75,717.317017,2024500,726.070007
716.609985,718.85498,703.539978,2090600,716.030029
731.450012,733.22998,713,2450900,714.469971
730.309998,738.5,719.059998,2963700,726.390015
730,747.179993,728.919983,1947000,743.619995
746.450012,752,738.640015,1950700,742.580017
743,744.059998,731.257996,3272800,741.840027
769.5,769.5,758.340027,1489600,758.880005
776.599976,777.599976,766.900024,1293300,771
766.690002,779.97998,766.429993,1765000,776.599976
752.919983,762.98999,749.52002,1515300,762.51001
749.549988,751.349976,746.619995,527200,748.400024
753.469971,754.210022,744,1565900,750.309998
751.650024,754.849976,745.530029,1365400,750
746.130005,750,740,1525700,747.77002
746.51001,754.130005,738.150024,3148700,739.309998
762.419983,762.679993,749,1553400,749.429993
750,760.590027,739.434998,1993300,758.090027
753,758.080017,743.01001,2666200,743.400024
741.789978,748.72998,724.169983,2412500,747.77002
741.159973,745.710022,736.75,2224400,738.869995
752.849976,755.849976,743.830017,1984900,749.460022
759.169983,764.22998,737.000977,2700000,751.609985
757.890015,764.799988,754.200012,1829500,762.369995
767.77002,768.72998,755.090027,1812300,763.25
753.099976,768.48999,750,2757300,766.809998
766.01001,768.994995,745.630005,2590600,752.539978
768.900024,775.955017,758.960022,2230400,762.380005
747.109985,768.950012,746.700012,2134600,767.039978
748.809998,754.929993,741.27002,2097600,742.599976
748.460022,753.409973,747.48999,838500,750.26001
748.140015,752,746.059998,1122100,748.150024
752,755.278992,737.630005,2333100,748.280029
757.450012,762.708008,751.820007,1414500,755.97998
746.530029,757.919983,743,2212300,756.599976
738.73999,742,737.429993,1327100,738.409973
727.580017,741.409973,727,1684300,740
729.289978,731.844971,723.026978,1510900,725.299988
715.599976,729.48999,711.330017,1905900,728.960022
729.169983,731.150024,716.72998,2075500,717
731,737.799988,728.64502,1837200,731.22998
732.460022,741,730.22998,1366400,735.400024
724.400024,730.590027,718.5,1608000,728.320007
730.200012,734.710022,719.429993,2069800,724.890015
731.5,735.409973,727.01001,1511600,733.76001
729.469971,739.47998,729.469971,1861600,731.25
722,733.099976,721.900024,1706700,728.109985
718.859985,724.650024,714.719971,1565400,722.159973
711.059998,721.619995,705.849976,1886300,721.109985
715.72998,718,710.049988,1908800,710.809998
710.5,718.26001,710.01001,1456000,716.919983
707.330017,712.97998,703.080017,2178900,712.950012
707.380005,713.619995,704.549988,2245800,708.48999
701.549988,719.150024,701.26001,2716600,712.780029
727.5,730,701.5,6653900,702
646.700012,657.799988,644.01001,4071000,651.789978
654.150024,655.869995,641.72998,1791100,642.609985
664.039978,664.719971,644.195007,2490000,650.280029
661.179993,666.820007,659.580017,1477300,666.099976
664.109985,664.969971,657.200012,1611100,662.200012
654.659973,663.130005,654.460022,1885700,661.73999
653.210022,659.390015,648.849976,1415500,651.159973
643.150024,657.812012,643.150024,1807700,652.299988
642.090027,648.5,639.01001,1275200,646.669983
640,645.98999,635.317993,1648700,643.609985
641.359985,644.450012,625.559998,2182100,639.159973
649.23999,650.609009,632.150024,2092700,642.359985
638.840027,649.25,636.530029,2166300,645.440002
632,643.01001,627,1803600,641.469971
607.200012,627.340027,603.130005,2684800,626.909973
608.369995,612.090027,599.849976,1867600,611.289978
603.280029,608.76001,600.72998,2413400,608.419983
597.280029,605,590.219971,2309500,594.969971
610.340027,614.60498,589.380005,3127700,594.890015
629.77002,629.77002,611,2174000,611.969971
616.640015,627.320007,612.400024,2240100,625.799988
622.049988,628.929993,620,1470900,622.359985
627,627.549988,615.429993,2562900,622.690002
634.400024,636.48999,625.940002,1788500,635.440002
636.789978,640,627.02002,5133400,629.25
637.789978,650.900024,635.02002,2274700,642.900024
635.469971,637.950012,632.320007,1286500,635.97998
626.700012,638.700012,623.780029,2084400,635.140015
625.700012,625.859985,619.429993,1702300,623.23999
619.75,625.780029,617.419983,1373500,625.77002
613.099976,624.159973,611.429993,1905300,621.349976
621.219971,626.52002,609.599976,1702100,612.719971
612.48999,616.309998,604.119995,2279500,614.659973
600,603.469971,595.25,2089000,600.700012
617,619.710022,602.820984,1759600,606.25
605.590027,614.340027,599.710022,2575600,614.340027
602.359985,612.859985,594.099976,3702100,597.789978
627.539978,635.799988,617.679993,2176700,618.25
632.820007,636.880005,624.559998,1978700,630.380005
639.400024,643.590027,622,3491300,637.609985
610.349976,631.710022,599.049988,4235900,628.619995
614.909973,617.450012,581.109985,3538000,582.059998
573,599.330017,565.049988,5770300,589.609985
639.780029,640.049988,612.330017,4265200,612.47998
655.460022,662.98999,642.900024,2855300,646.830017
656.599976,667,654.190002,2131600,660.900024
661.900024,664,653.460022,1456100,656.130005
656.799988,661.380005,651.23999,1051700,660.869995
655.01001,659.85498,652.659973,1072100,657.119995
659.322021,664.5,651.661011,1810700,656.450012
663.080017,665,652.289978,2936700,659.559998
669.200012,674.900024,654.27002,5029200,660.780029
639.47998,643.440002,631.249023,1809200,633.72998
640.22998,642.679993,629.710022,1403900,635.299988
645,645.379028,632.25,1572600,642.679993
634.330017,647.859985,633.159973,2334300,643.780029
628.419983,634.809998,627.159973,1490900,629.25
625.340027,633.05603,625.340027,1304500,631.210022
631.380005,632.909973,625.5,1706100,625.609985
630,635.219971,622.049988,1474200,632.590027
628.799988,633.359985,622.650024,1575100,631.929993
632.830017,632.830017,623.309998,1727300,628
621,634.299988,620.5,2675400,627.26001
647,648.169983,622.52002,3625700,623.559998
661.27002,663.630005,641,3029100,644.280029
660.890015,678.640015,659,3929300,662.099976
655.210022,673,654.299988,3377200,662.299988
659.23999,668.880005,653.01001,5860900,663.02002
649,674.468018,645,11164900,672.929993
565.119995,580.679993,565,4768300,579.849976
560.130005,566.502991,556.789978,1784600,560.219971
546.76001,565.848999,546.710022,3244100,561.099976
532.880005,547.109985,532.400024,2206500,546.549988
526.289978,532.559998,525.549988,1956700,530.130005
523.119995,523.77002,520.349976,1839400,520.679993
521.049988,522.734009,516.109985,1296700,516.830017
523.130005,526.179993,515.179993,1597200,525.02002
519.5,525.25,519,1280500,522.859985
521.080017,524.650024,521.080017,1235900,523.400024
524.72998,525.690002,518.22998,1961000,521.840027
526.02002,526.25,520.5,2217200,520.51001
525.01001,528.609985,520.539978,1937800,521.52002
537.26001,537.76001,531.349976,2109100,531.690002
538.869995,540.900024,535.22998,1335700,535.22998
540,540,535.659973,1286600,537.840027
539.640015,541.499023,535.25,1197500,540.47998
539.590027,543.73999,537.530029,1250300,538.190002
537.210022,538.25,533.01001,1893500,536.690002
531,538.150024,530.789978,1833100,536.72998
529.369995,530.97998,525.099976,1294200,529.26001
528.400024,529.640015,525.559998,1071800,528.150024
528,528.299988,524,1632700,527.200012
531.599976,533.119995,530.159973,955800,532.330017
538.424988,538.97998,533.02002,1217500,534.609985
529.359985,538.359985,529.349976,1815000,536.690002
527.559998,529.200012,523.01001,1455300,526.690002
533.309998,534.119995,526.23999,1520600,526.830017
536.349976,537.200012,532.52002,1388200,533.330017
537.76001,540.590027,534.320007,1348300,536.700012
539.909973,543.5,537.109985,1714500,540.309998
532.929993,543,531.330017,1939000,539.179993
536.789978,536.789978,529.76001,1904300,533.98999
537.369995,538.630005,531.450012,2597400,532.109985
538.01001,540.609985,536.25,1029800,539.780029
532.799988,540.549988,531.710022,1525000,539.789978
538.119995,539,529.880005,2406500,532.320007
540.150024,544.190002,539.51001,1176200,540.109985
537.950012,543.840027,535.97998,1462700,542.51001
538.48999,542.919983,532.971985,1430800,539.27002
533.97998,540.659973,533.039978,1966900,537.359985
532.01001,534.820007,528.849976,2003400,532.299988
539.179993,539.273987,530.380005,1971300,533.849976
533.77002,539,532.409973,1403900,538.400024
530.559998,534.322021,528.655029,1252300,529.619995
531.599976,533.208984,525.26001,1634200,529.039978
538.369995,541.97998,535.400024,905300,535.700012
536.650024,541.150024,525,1527600,538.219971
523.98999,533.460022,521.75,1546300,530.700012
531.23999,532.380005,521.085022,1567000,524.219971
538.210022,539.73999,530.390991,1383100,530.799988
538.530029,544.070007,535.059998,1308000,540.780029
538.429993,539.539978,532.099976,1768200,537.900024
547.869995,548.590027,535.049988,2082200,537.340027
550.469971,553.679993,546.905029,1698800,549.080017
554.640015,556.02002,550.366028,1491000,553.679993
563.390015,565.950012,553.200012,2398000,555.369995
566.102584,571.14259,557.252507,4932500,565.062561
541.002435,550.96249,540.23244,4184800,547.002472
534.402426,541.082428,531.752397,1593500,539.367458
537.512456,539.392429,533.677415,1844700,533.972413
525.602352,536.092424,524.50235,1679200,535.382408
528.662379,529.842373,521.012371,2151800,524.052386
529.902414,535.592396,529.612373,1299800,533.802391
528.702406,534.732432,523.22235,2318800,532.532429
536.252409,537.572435,528.094416,2604100,530.392405
538.412447,544.062463,537.312445,1645300,539.172466
542.292472,542.292472,537.312445,1409400,540.012416
541.032425,541.952428,535.492451,1557800,540.782472
538.382457,543.852476,538.382457,1178500,541.612446
538.08244,542.692434,536.002456,1302800,537.022404
532.222436,538.412447,529.572407,1324400,536.767432
540.852427,540.852427,533.849395,1716300,535.532417
548.602502,551.142488,539.502472,1963000,542.562439
550.00246,554.712521,546.722468,1588000,548.002468
551.622503,553.472487,548.17249,1287500,552.032502
553.002509,555.282504,548.132463,1897400,548.342512
557.59255,558.90254,550.652497,1572600,555.172522
570.50259,572.262605,558.742555,2152200,558.787539
562.562541,574.592603,561.212525,2583200,570.192597
560.432554,562.362529,555.832536,1643800,558.81251
561.652513,561.722529,559.052548,2616800,560.362537
559.392531,560.802526,556.147548,1197200,557.992512
552.50248,559.782516,547.002472,2134500,559.502513
551.712472,553.802493,548.002468,1805500,550.842471
550.952514,556.852545,546.002476,1640900,554.512509
553.502476,558.402511,544.222448,1703500,547.322503
553.512513,556.37253,550.462462,1389600,555.512505
555.142533,558.142521,550.682486,1820700,551.182515
564.252539,564.852573,554.732534,1792300,555.012538
566.862541,570.272589,563.537566,1062100,568.852557
574.882583,576.682625,566.762536,1659100,567.687558
575.022616,577.912621,573.412609,1389600,575.332609
571.872619,577.112637,568.012546,1876800,573.372583
570.452587,575.392588,566.522559,1704700,573.64261
560.532559,572.152562,558.752531,2129600,571.342601
554.242482,564.712541,552.902503,2410100,558.402511
543.212476,556.142529,541.502464,2311500,555.482516
535.90245,546.222501,535.447406,1825900,543.872489
530.002419,536.792403,528.252381,1005000,536.092424
536.052398,536.441404,529.412422,1457800,531.912381
543.132484,543.75247,535.802444,1444300,538.952441
538.042413,543.11247,538.012424,989100,542.872432
541.402458,545.492471,537.512456,1453000,539.702422
546.83245,550.00246,541.092465,1616800,542.842443
543.352447,549.912491,543.132484,1900300,549.012501
537.252405,544.822482,534.675391,1620200,542.932472
535.302416,538.452412,533.380397,1377700,535.972405
529.302379,537.702431,526.922378,1749800,536.942412
528.002366,532.002411,526.022388,1267700,527.832406
527.64237,537.202402,526.412373,1763500,531.002415
523.792395,528.502395,522.092359,1849700,527.582391
529.2424,532.67442,521.272361,1663600,522.762349
528.002366,533.40243,523.262377,2038600,529.2424
531.732383,533.002407,518.552377,2849800,528.482381
515.862322,539.872444,515.522339,5606300,534.522445
511.002313,511.092313,501.202274,4186300,510.6623
522.782362,522.992349,510.002318,1683800,510.002318
529.972369,530.702398,518.19232,1904000,518.63237
538.532466,539.002444,529.672413,1543700,535.212448
535.592396,542.172453,533.002407,2281700,539.952437
521.482349,536.332401,519.702382,2676900,534.392388
507.252283,519.282346,506.202284,2268700,518.042373
511.002313,512.502307,506.018277,2232000,506.902294
500.012273,508.1923,500.002267,2298200,508.082288
505.572291,505.682303,497.762267,2715800,501.792271
494.652237,503.232286,493.002234,2215500,500.872267
498.842256,502.982272,492.392224,2370400,496.182251
494.942247,495.97823,487.562205,2326700,492.552239
504.7623,504.922285,494.792239,2071300,496.172244
497.992268,503.48227,491.002212,3353500,502.682285
507.002299,507.246285,499.652247,2065000,501.102268
515.002358,516.177334,501.052266,2899900,501.962262
523.262377,524.332389,513.062315,2059800,513.872306
529.012399,531.272382,524.102388,1447500,524.812404
531.252429,532.602384,525.802363,1368200,526.402397
528.092396,531.152424,527.132366,876200,530.422394
532.192385,535.482414,530.013375,2278500,530.332426
528.772422,534.252417,527.312364,1040500,534.032392
530.512424,531.761394,527.022384,705900,528.772422
527.00237,534.56241,526.292354,2197600,530.592416
516.082347,526.462376,516.082347,2723700,524.872383
511.512318,517.722342,506.91328,3690200,516.352313
512.952333,513.872306,504.70229,2926700,511.102319
497.002248,507.002299,496.812244,2883100,504.892295
511.562321,513.052308,489.00222,3964200,495.392242
522.742396,523.102392,513.272333,2813400,513.802351
523.512391,528.502395,518.662359,1994500,518.662359
527.802416,533.922411,527.102376,1610800,528.34241
533.082399,536.332401,525.562386,1712200,526.062353
522.142362,534.192438,520.502366,1871300,533.37244
527.132366,531.002415,523.792395,2329300,526.982357
531.002415,532.892425,524.282386,2565600,525.262369
531.1624,537.342434,528.592363,1392100,537.312445
531.442404,535.998416,529.262414,1277900,531.322384
533.512412,535.502427,529.802408,1526600,533.752389
538.902438,541.412434,531.862379,2115300,533.802391
540.622426,542.002431,536.602429,1148300,541.832471
540.882478,541.552467,537.044437,1522900,540.372473
539.002444,543.982471,538.60444,1789900,541.082428
537.652428,542.702471,535.622446,1706300,539.272471
541.612446,542.142464,536.562402,2224200,537.502419
531.252429,535.112442,531.082408,1563300,534.832438
535.002399,538.242425,530.082412,1392200,536.992415
537.502419,541.942452,534.172425,1962600,535.032449
543.582448,543.792436,534.063422,1726000,536.512399
546.682503,546.682503,542.15244,1289500,544.402446
549.80251,549.80251,543.482442,1339300,545.38249
550.392507,550.462462,545.172441,1129600,547.312465
548.492459,551.942473,546.302493,965500,550.292501
541.462437,549.592461,541.022449,1134600,547.492463
546.212464,546.212464,538.672437,1633700,541.012473
545.502448,546.887472,540.972446,1333200,542.042458
556.802542,556.802542,544.052487,2032200,545.922484
553.002509,555.502529,549.302481,1244200,554.112486
555.502529,557.902544,553.23251,1382200,555.222525
559.352504,559.572529,554.752486,2035000,559.082538
548.952461,552.802497,543.512432,1455600,550.312514
550.00246,554.192479,546.982459,1770500,549.33247
543.002488,548.982512,541.622422,1270900,548.902458
537.032441,544.412483,537.032441,1185300,540.772435
544.36248,544.882461,535.792407,1973100,539.782415
539.322413,547.222497,535.852447,2348800,543.982471
529.892376,539.802428,528.802412,2919200,532.712427
525.192353,526.792383,519.112324,2336200,526.542369
509.452317,521.762353,508.102301,2607500,520.842349
527.252385,530.982402,508.532313,5539300,511.172305
519.002342,529.432374,515.002358,3708500,524.512387
531.012391,532.802396,518.302363,3719300,530.032409
538.902438,547.192507,533.172429,2222500,537.942408
544.992443,549.502492,533.102413,2581600,533.212394
557.722546,565.132577,544.052487,3081900,544.492476
571.182555,571.49261,559.062524,2524800,560.882518
565.572566,573.882587,557.492545,1990800,572.502582
574.402629,575.27263,563.742534,1911300,563.742534
578.802636,581.002639,574.442595,1214600,577.352614
573.052613,577.227637,572.502582,1141700,575.282606
567.312567,571.912585,563.322559,1178400,570.082554
576.012635,577.582615,567.01255,1445400,568.272597
576.932639,579.852634,572.852602,1621700,577.36259
571.7526,578.192625,571.172579,1282400,576.362594
576.062638,579.2526,574.662619,1443600,577.1026
587.552646,587.982658,574.182604,1925900,575.062581
581.462641,589.632691,580.522624,1728100,587.992634
586.852667,586.852667,581.002639,1471400,581.132634
593.82271,593.951665,583.462632,1689500,587.372648
591.502688,596.482715,589.502696,3736600,596.082692
587.002675,589.542661,585.002622,1444500,589.272695
580.012619,587.522656,578.777604,1692800,584.772622
572.762572,581.502606,572.662566,1480300,579.95264
572.94257,574.952599,568.212557,1597500,573.102616
581.002639,581.642639,574.462608,1601600,575.622589
580.362639,581.812661,576.262588,1221000,581.352659
581.502606,583.502659,576.942615,977300,583.102636
588.902662,589.002667,580.002643,1287200,581.012615
586.602653,591.772654,586.302635,1431000,589.722659
583.982613,586.55265,581.952632,1632300,586.082672
580.002643,586.00268,579.222611,1458100,581.982621
580.002643,582.992655,575.002602,1215100,577.942611
571.852606,577.832629,571.192593,1578400,577.332601
571.332564,572.04258,567.07155,1083700,571.602592
569.562573,573.252625,567.102579,1292900,569.202577
577.272622,578.492642,570.105566,1703300,571.002557
581.262629,581.802623,576.582619,1639600,577.862619
584.722619,585.002622,579.002647,1361400,580.202654
583.592628,585.238621,580.642643,789000,582.562642
583.822628,584.502655,581.14261,914800,583.372664
585.88266,586.702658,582.572618,1036700,584.492618
585.002622,587.342658,584.002627,978600,586.862643
576.11258,584.512631,576.002598,1284100,582.162619
577.862619,579.382595,570.522603,1519100,573.482626
576.182596,577.902645,570.882599,985400,574.652582
567.312567,575.002602,565.752564,1439200,574.782577
564.522567,565.902572,560.882518,1542000,562.732562
569.992585,570.492553,566.002578,1214700,567.882551
563.562536,570.252576,560.352561,1494700,568.772565
568.00257,569.89258,561.102543,1110900,563.362525
561.782569,570.702601,560.002541,1334300,566.376589
570.052564,571.982601,562.612543,1551200,565.072537
569.042592,575.352622,564.102531,1427300,573.152619
570.402584,575.962633,562.85252,1955200,566.072533
580.602616,583.652668,570.002561,2102700,571.602592
586.55265,589.502696,584.002627,1016400,587.42265
588.752653,589.702646,583.517654,1349800,585.612633
588.072688,592.502683,584.755668,986800,590.602697
590.402686,591.862684,587.032665,932400,589.022681
596.452725,599.502716,591.772654,1035100,593.352671
593.232713,597.852683,592.502683,1233100,595.982686
590.722655,599.652725,590.602697,1699100,594.742713
591.752702,594.40267,585.235622,2062100,589.472645
593.002712,596.802684,582.002635,4014100,595.082696
579.532604,580.992601,568.61258,3016500,573.732579
588.002671,588.402633,582.202646,1397100,582.662648
585.742628,585.807626,576.562606,1623000,584.782659
582.602608,585.212671,578.032641,1854000,584.872627
571.912585,580.85263,571.422594,1621700,579.182645
565.912548,576.592595,565.012558,1356700,571.102563
571.582578,576.72259,569.378597,1116700,576.08259
577.662607,579.530645,566.137592,1909500,571.092587
583.76265,586.432631,579.592644,1064600,582.252649
583.352651,585.01266,580.922646,714100,584.732656
583.352651,585.442672,580.392628,1056300,582.33766
578.32262,584.402649,576.652635,1447900,582.672624
578.662603,579.572631,574.752588,1313700,575.282606
577.182592,579.872648,573.802595,2236900,577.242632
581.002639,582.45266,571.852606,1741900,576.002598
565.262572,579.962616,565.222546,1969300,578.652627
565.192556,572.648612,561.012513,2207100,564.622572
555.152509,565.002582,554.252519,1536800,564.952579
556.852545,557.582513,550.394465,4508300,556.362492
554.242482,555.0025,548.512473,2456800,554.902495
544.862448,553.562516,544.002484,1741700,553.372481
544.202435,545.32245,539.33245,1444500,543.012465
549.262515,549.622511,541.522477,1702500,544.282488
552.262503,552.30253,545.562488,1220500,551.762475
557.302509,557.992512,548.46247,1458400,551.352476
558.002549,559.882522,555.022514,1100100,558.842499
560.512546,563.602563,557.902544,1351700,560.552511
557.152501,562.902523,556.042523,1467500,562.122552
558.062528,558.062528,548.932509,1736700,556.332503
546.402499,554.952498,544.452449,1689100,553.902499
541.502464,548.612478,538.752429,1816400,544.662436
550.99248,552.342495,542.552463,1866600,544.94244
560.70252,560.902531,545.732448,1435000,553.932488
560.802526,561.352557,555.912528,1771100,559.892559
563.352549,564.002525,558.712504,1354100,560.082534
564.57257,567.842585,561.002537,1652000,561.682564
556.002496,566.002578,554.352525,2104100,565.952575
547.262462,553.642508,543.702467,1932100,552.702492
541.132431,547.602506,540.782472,1615800,545.062459
532.902401,539.185441,531.912381,1196200,538.942465
529.742368,536.232457,526.302392,1784700,529.772418
519.702382,529.782394,517.58537,1277700,528.862391
521.39238,521.802379,515.442347,1485200,520.632362
525.702357,525.872379,517.422325,1704300,519.982324
533.002407,533.002407,525.292358,1191700,526.652412
530.892433,536.072411,529.512367,1653400,533.092437
523.512391,530.192393,519.012379,1912500,529.922427
510.752299,519.902332,504.202292,2439400,518.732375
508.462297,517.232351,506.452298,2021300,511.002313
515.792305,516.68232,503.302272,3224300,509.962321
525.232379,526.812396,515.062337,1689000,515.14233
524.822381,528.902418,521.322364,1024100,527.812392
533.762426,534.002403,525.612389,1688500,527.932411
527.112413,532.932391,523.882364,1905500,531.352435
527.602405,528.002366,522.522372,1751100,526.662388
516.902344,529.462425,516.322324,2699000,527.70241
517.182348,518.602319,502.802274,3335500,517.152359
522.512395,524.702361,515.422333,2100300,516.182352
530.072374,531.652391,522.122349,1883100,525.162363
533.792415,533.872408,526.252389,2052300,526.942391
528.642366,537.232453,527.512375,2365300,534.812425
536.1024,536.702435,525.602352,2566700,528.622414
548.81249,549.502492,531.152424,6809500,536.1024
543.002488,557.002492,540.00244,4893200,556.54249
536.822454,538.452412,518.462348,3855000,536.442444
538.252462,544.10249,529.56237,2575000,532.522392
532.552381,540.00244,526.532392,3924800,530.602392
565.002582,565.002582,539.902434,4036800,540.952433
559.622532,565.372554,552.952506,3330800,564.142557
542.602466,555.0025,541.612446,3151200,554.902495
540.742445,548.482483,527.152379,4401600,538.152456
574.652582,577.772589,543.002488,6369300,543.14246
569.852553,587.282679,564.132581,5099100,569.742571
599.992707,604.832763,562.192568,147100,567.002574
558.712504,568.452595,558.712504,7900,567.162558
566.892592,567.002574,556.932537,10800,556.972503
561.202549,566.43259,558.672539,41200,559.992565
568.00257,568.00257,552.922516,13100,558.462551


================================================
FILE: data_loader.py
================================================
"""Data loader.

Author: Jinsung Yoon
Contact: jsyoon0823@gmail.com
----------------------------------------
Loads Google stock dataset with MinMax normalization.
Reference: https://finance.yahoo.com/quote/GOOGL/history?p=GOOGL
"""

# Necessary Packages
import numpy as np
from utils import MinMaxScaler


def data_loader(train_rate = 0.8, seq_len = 7):
  """Loads Google stock data.
  
  Args:
    - train_rate: the ratio between training and testing sets
    - seq_len: sequence length
    
  Returns:
    - train_x: training feature
    - train_y: training labels
    - test_x: testing features
    - test_y: testing labels
  """
  
  # Load data
  ori_data = np.loadtxt('data/google.csv', delimiter=',', skiprows = 1)
  # Reverse the time order
  reverse_data = ori_data[::-1]
  # Normalization
  norm_data = MinMaxScaler(reverse_data)
    
  # Build dataset
  data_x = []
  data_y = []
  
  for i in range(0, len(norm_data[:,0]) - seq_len):
    # Previous seq_len data as features
    temp_x = norm_data[i:i + seq_len,:]
    # Values at next time point as labels
    temp_y = norm_data[i + seq_len, [-1]]
    data_x = data_x + [temp_x]
    data_y = data_y + [temp_y]
    
  data_x = np.asarray(data_x)
  data_y = np.asarray(data_y)
            
  # Train / test Division   
  idx = np.random.permutation(len(data_x))
  train_idx = idx[:int(train_rate * len(data_x))]
  test_idx = idx[int(train_rate * len(data_x)):]
        
  train_x, test_x = data_x[train_idx, :, :], data_x[test_idx, :, :]
  train_y, test_y = data_y[train_idx, :], data_y[test_idx, :]
    
  return train_x, train_y, test_x, test_y


================================================
FILE: main_time_series_prediction.py
================================================
"""Time-series prediction main function

Author: Jinsung Yoon
Contact: jsyoon0823@gmail.com
------------------------------------
(1) Load data
(2) Train model (RNN, GRU, LSTM, Attention)
(3) Evaluate the trained model
"""

# Necessary packages
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import warnings
warnings.filterwarnings("ignore")

from data_loader import data_loader
from basic_rnn_lstm_gru import GeneralRNN
from basic_attention import Attention
from utils import performance


def main (args):  
  """Time-series prediction main function.
  
  Args:
    - train_rate: training data ratio
    - seq_len: sequence length
    - task: classification or regression
    - model_type: rnn, lstm, gru, or attention
    - h_dim: hidden state dimensions
    - n_layer: number of layers
    - batch_size: the number of samples in each mini-batch
    - epoch: the number of iterations
    - learning_rate: learning rates
    - metric_name: mse or mae
  """
  # Load data
  train_x, train_y, test_x, test_y = data_loader(args.train_rate, 
                                                 args.seq_len)
  
  # Model traininig / testing
  model_parameters = {'task': args.task,
                      'model_type': args.model_type,
                      'h_dim': args.h_dim,
                      'n_layer': args.n_layer,
                      'batch_size': args.batch_size,
                      'epoch': args.epoch,
                      'learning_rate': args.learning_rate}
  
  if args.model_type in ['rnn','lstm','gru']:
    general_rnn = GeneralRNN(model_parameters)    
    general_rnn.fit(train_x, train_y)
    test_y_hat = general_rnn.predict(test_x)
  elif args.model_type == 'attention':
    basic_attention = Attention(model_parameters)    
    basic_attention.fit(train_x, train_y)
    test_y_hat = basic_attention.predict(test_x)
  
  # Evaluation
  result = performance(test_y, test_y_hat, args.metric_name)
  print('Performance (' + args.metric_name + '): ' + str(result))
  
    
##  
if __name__ == '__main__':
  
  # Inputs for the main function
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--train_rate',
      help='training data ratio',
      default=0.8,
      type=str)
  parser.add_argument(
      '--seq_len',
      help='sequence length',
      default=7,
      type=int)
  parser.add_argument(
      '--model_type',
      choices=['rnn','gru','lstm','attention'],
      default='attention',
      type=str)
  parser.add_argument(
      '--h_dim',
      default=10,
      type=int)
  parser.add_argument(
      '--n_layer',
      default=3,
      type=int)
  parser.add_argument(
      '--batch_size',
      default=32,
      type=int)
  parser.add_argument(
      '--epoch',
      default=100,
      type=int)
  parser.add_argument(
      '--learning_rate',
      default=0.01,
      type=float)
  parser.add_argument(
      '--task',
      choices=['classification','regression'],
      default='regression',
      type=str)
  parser.add_argument(
      '--metric_name',
      choices=['mse','mae'],
      default='mae',
      type=str)
  
  args = parser.parse_args() 
  
  # Call main function  
  main(args)

================================================
FILE: utils.py
================================================
"""Utility functions for time-series prediction.

Author: Jinsung Yoon
Contact: jsyoon0823@gmail.com
------------------------------------
(1) MinMaxScaler: MinMax normalizer
(2) performance: performance evaluator
(3) binary_cross_entropy_loss: loss for RNN on classification
(4) mse_loss: loss for RNN on regression
(5) rnn_sequential: Architecture
"""

# Necessary packages
import numpy as np
from sklearn.metrics import mean_squared_error, mean_absolute_error
import tensorflow as tf
from tensorflow.keras import layers


def MinMaxScaler(data):    
  """Normalizer (MinMax criteria).
  
  Args:
    - data: original data
    
  Returns:
    - norm_data: normalized data
  """    
  numerator = data - np.min(data, 0)
  denominator = np.max(data, 0) - np.min(data, 0)
  norm_data = numerator / (denominator + 1e-8)
  return norm_data


def performance(test_y, test_y_hat, metric_name):
  """Evaluate predictive model performance.
  
  Args:
    - test_y: original testing labels
    - test_y_hat: prediction on testing data
    - metric_name: 'mse' or 'mae'
    
  Returns:
    - score: performance of the predictive model
  """  
  assert metric_name in ['mse', 'mae']
  
  if metric_name == 'mse':
    score = mean_squared_error(test_y, test_y_hat)
  elif metric_name == 'mae':
    score = mean_absolute_error(test_y, test_y_hat)
    
  score = np.round(score, 4)
    
  return score


def binary_cross_entropy_loss (y_true, y_pred):
  """User defined cross entropy loss.
  
  Args:
    - y_true: true labels
    - y_pred: predictions
    
  Returns:
    - loss: computed loss
  """
  # Exclude masked labels
  idx = tf.cast((y_true >= 0), float)
  # Cross entropy loss excluding masked labels
  loss = -(idx * y_true * tf.math.log(y_pred) + \
           idx * (1-y_true) * tf.math.log(1-y_pred))
  return loss


def mse_loss (y_true, y_pred):
  """User defined mean squared loss.
  
  Args:
    - y_true: true labels
    - y_pred: predictions
    
  Returns:
    - loss: computed loss
  """
  # Exclude masked labels
  idx = tf.cast((y_true >= 0), float)
  # Mean squared loss excluding masked labels
  loss = idx * tf.pow(y_true - y_pred, 2)
  return loss


def rnn_sequential (model, model_name, h_dim, return_seq):
  """Add one rnn layer in sequential model.
  
  Args:
    - model: sequential rnn model
    - model_name: rnn, lstm, or gru
    - h_dim: hidden state dimensions
    - return_seq: True or False
    
  Returns:
    - model: sequential rnn model
  """
  
  if model_name == 'rnn':
    model.add(layers.SimpleRNN(h_dim, return_sequences=return_seq))
  elif model_name == 'lstm':
    model.add(layers.LSTM(h_dim, return_sequences=return_seq))
  elif model_name == 'gru':
    model.add(layers.GRU(h_dim, return_sequences=return_seq))
    
  return model