[
  {
    "path": ".github/FUNDING.yml",
    "content": "# These are supported funding model platforms\n\nko_fi: omegajoctan\n\ncustom: ['https://www.mql5.com/en/users/omegajoctan/seller']\ncustom: ['https://www.buymeacoffee.com/omegajoctan']\n"
  },
  {
    "path": ".gitignore",
    "content": "*.ex5\n*.psd\n*.zip\n*.rar\n\n*.xlsx\n\nTodo's.txt\nlogisticwiki.txt\n\n/venv\n\n/Neural Nets Pro"
  },
  {
    "path": "Examples/Classifier Model Example.mq5",
    "content": "//+------------------------------------------------------------------+\n//|                                      Classifier Model Sample.mq5 |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n#property version   \"1.00\"\n\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\preprocessing.mqh>\n#include <MALE5\\MatrixExtend.mqh> //helper functions for for data manipulations\n#include <MALE5\\metrics.mqh> //fo measuring the performance\n\nStandardizationScaler scaler; //standardization scaler from preprocessing.mqh\nCDecisionTreeClassifier *decision_tree; //a decision tree classifier model\n\nMqlRates rates[];\n//+------------------------------------------------------------------+\n//| Expert initialization function                                   |\n//+------------------------------------------------------------------+\nint OnInit()\n  {\n  \n//--- Model selection\n   \n     decision_tree = new CDecisionTreeClassifier(2, 5); //a decision tree classifier from DecisionTree class\n     \n//---\n\n     vector open, high, low, close;     \n     int data_size = 1000;\n     \n//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1\n     \n     open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);\n     high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);\n     low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);\n     close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);\n     \n     matrix X(data_size, 3); //creating the x matrix \n   \n//--- Assigning the open, high, and low price values to the x matrix \n\n     X.Col(open, 0);\n     X.Col(high, 1);\n     X.Col(low, 2);\n     \n//--- Since we are using the x variables to predict y, we choose the close price to be the target variable \n   \n     vector y(data_size); \n     for (int i=0; i<data_size; i++)\n       {\n         if (close[i]>open[i]) //a bullish candle appeared\n           y[i] = 1; //buy signal\n         else\n           {\n             y[i] = 0; //sell signal\n           } \n       }\n\n//--- We split the data into training and testing samples for training and evaluation\n \n     matrix X_train, X_test;\n     vector y_train, y_test;\n     \n     double train_size = 0.7; //70% of the data should be used for training the rest for testing\n     int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable\n      \n     MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples         \n     \n\n//--- Normalizing the independent variables\n   \n     X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether\n     X_test = scaler.transform(X_test); // we transform the new data this way\n     \n\n//--- Training the  model\n     \n     decision_tree.fit(X_train, y_train); //The training function \n     \n//--- Measuring predictive accuracy \n   \n     vector train_predictions = decision_tree.predict_bin(X_train);\n     \n     Print(\"Training results classification report\");\n     Metrics::classification_report(y_train, train_predictions);\n\n//--- Evaluating the model on out-of-sample predictions\n     \n     vector test_predictions = decision_tree.predict_bin(X_test);\n     \n     Print(\"Testing results classification report\");\n     Metrics::classification_report(y_test, test_predictions); \n     \n//---\n\n    ArraySetAsSeries(rates, true);\n        \n\n   return(INIT_SUCCEEDED);\n  }\n//+------------------------------------------------------------------+\n//| Expert deinitialization function                                 |\n//+------------------------------------------------------------------+\nvoid OnDeinit(const int reason)\n  {\n//---\n    delete (decision_tree); //We have to delete the AI model object from the memory\n  }\n//+------------------------------------------------------------------+\n//| Expert tick function                                             |\n//+------------------------------------------------------------------+\nvoid OnTick()\n  {\n     \n//--- Making predictions live from the market \n   \n   CopyRates(Symbol(), PERIOD_D1, 1, 3, rates); //Get the very recent information from the market\n   \n   vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data\n   \n   x = scaler.transform(x);\n   int signal = (int)decision_tree.predict_bin(x);\n   \n   Comment(\"Signal = \",signal==1?\"BUY\":\"SELL\");  //Ternary operator for checking if the signal is either buy or sell\n  }\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Examples/Regressor Model Example.mq5",
    "content": "//+------------------------------------------------------------------+\n//|                                       Regressor Model sample.mq5 |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n#property version   \"1.00\"\n\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\preprocessing.mqh>\n#include <MALE5\\MatrixExtend.mqh> //helper functions for for data manipulations\n#include <MALE5\\metrics.mqh> //fo measuring the performance\n\nStandardizationScaler scaler; //standardization scaler from preprocessing.mqh\nCDecisionTreeRegressor *decision_tree; //a decision tree classifier model\n\nMqlRates rates[];\n//+------------------------------------------------------------------+\n//| Expert initialization function                                   |\n//+------------------------------------------------------------------+\nint OnInit()\n  {\n\n//--- Model selection\n   \n     decision_tree = new CDecisionTreeRegressor(2, 5); //a decision tree classifier from DecisionTree class\n\n\n     vector open, high, low, close;     \n     int data_size = 1000; //bars\n     \n//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1\n     \n     open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);\n     high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);\n     low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);\n     \n     close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);\n     \n     matrix X(data_size, 3); //creating the x matrix \n   \n//--- Assigning the open, high, and low price values to the x matrix \n\n     X.Col(open, 0);\n     X.Col(high, 1);\n     X.Col(low, 2);\n     \n     vector y = close; // The target variable is the close price, using open, high and low values were want to predict the next closing price\n     \n//--- We split the data into training and testing samples for training and evaluation\n \n     matrix X_train, X_test;\n     vector y_train, y_test;\n     \n     double train_size = 0.7; //70% of the data to be used for training the rest 30% for testing\n     int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable\n      \n     MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples         \n     \n//--- Normalizing the independent variables\n   \n     X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether\n     X_test = scaler.transform(X_test); // we transform the new data this way\n     \n//--- Training the  model\n     \n     decision_tree.fit(X_train, y_train); //The training function \n     \n//--- Measuring predictive accuracy \n   \n     vector train_predictions = decision_tree.predict(X_train);\n     \n     printf(\"Decision decision_tree training r2_score = %.3f \",Metrics::RegressionMetric(y_train, train_predictions, METRIC_R_SQUARED));\n\n//--- Evaluating the model on out-of-sample predictions\n     \n     vector test_predictions = decision_tree.predict(X_test);\n     \n     printf(\"Decision decision_tree out-of-sample r2_score = %.3f \",Metrics::r_squared(y_test, test_predictions)); \n\n\n   return(INIT_SUCCEEDED);\n  }\n//+------------------------------------------------------------------+\n//| Expert deinitialization function                                 |\n//+------------------------------------------------------------------+\nvoid OnDeinit(const int reason)\n  {\n//---\n    delete (decision_tree); //We have to delete the AI model object from the memory\n  }\n//+------------------------------------------------------------------+\n//| Expert tick function                                             |\n//+------------------------------------------------------------------+\nvoid OnTick()\n  {\n     \n//--- Making predictions live from the market \n   \n   CopyRates(Symbol(), PERIOD_D1, 1, 3, rates); //Get the very recent information from the market\n   \n   vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data\n   \n   x  = scaler.transform(x);\n   double predicted_close_price = decision_tree.predict(x);\n   \n   Comment(\"Next closing price predicted is = \",predicted_close_price);  \n  }\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "LICENSE",
    "content": "MIT License\n\nCopyright (c) 2023 Omega Joctan\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "MqPlotLib/plots.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                        plots.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include <Graphics\\Graphic.mqh>\n#include <MALE5\\MatrixExtend.mqh>\n\nclass CPlots\n  {  \nprotected:\n   CGraphic *graph;\n   \n   long m_chart_id;\n   int m_subwin;\n   int m_x1, m_x2;\n   int m_y1, m_y2;\n   string m_font_family;\n   bool m_chart_show;\n   \n   string m_plot_names[];\n   ENUM_CURVE_TYPE m_curve_type;\n   bool GraphCreate(string plot_name);\n   \n   vector m_x, m_y;\n   string x_label, y_label;\n   \npublic:\n         CPlots(long chart_id=0, int sub_win=0 ,int x1=30, int y1=40, int x2=550, int y2=310, string font_family=\"Consolas\", bool chart_show=true);\n        ~CPlots(void);\n         \n         bool Plot(string plot_name, vector& x, vector& y, string x_axis_label, string y_axis_label, string label, ENUM_CURVE_TYPE curve_type=CURVE_POINTS_AND_LINES,color  clr = clrDodgerBlue, bool   points_fill = true);\n         bool AddPlot(vector &v,string label=\"plt\",color clr=clrOrange);\n  };\n  \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPlots::CPlots(long chart_id=0, int sub_win=0 ,int x1=30, int y1=40, int x2=550, int y2=310, string font_family=\"Consolas\", bool chart_show=true):\n   m_chart_id(chart_id),\n   m_subwin(sub_win),\n   m_x1(x1),\n   m_y1(y1),\n   m_x2(x2),\n   m_y2(y2),   \n   m_font_family(font_family),\n   m_chart_show(chart_show)\n {\n   graph = new CGraphic();\n   ChartRedraw(m_chart_id);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPlots::~CPlots(void)\n {\n   for (int i=0; i<ArraySize(m_plot_names); i++)\n       ObjectDelete(m_chart_id,m_plot_names[i]);\n   \n   delete(graph);\n   ChartRedraw();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CPlots::GraphCreate(string plot_name)\n {   \n   ChartRedraw(m_chart_id);\n   \n   ArrayResize(m_plot_names,ArraySize(m_plot_names)+1);\n   m_plot_names[ArraySize(m_plot_names)-1] = plot_name;\n   ChartSetInteger(m_chart_id, CHART_SHOW, m_chart_show);\n   \n   if(!graph.Create(m_chart_id, plot_name, m_subwin, m_x1, m_y1, m_x2, m_y2))\n     {\n      printf(\"Failed to Create graphical object on the Main chart Err = %d\", GetLastError());\n      return(false);\n     }\n     \n   return (true);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nbool CPlots::Plot(\n                  string plot_name,\n                  vector& x,\n                  vector& y,\n                  string x_axis_label,\n                  string y_axis_label,\n                  string label,\n                  ENUM_CURVE_TYPE curve_type=CURVE_POINTS_AND_LINES,\n                  color  clr = clrDodgerBlue,\n                  bool   points_fill = true\n               )\n  {\n   \n   if (!this.GraphCreate(plot_name))\n     return (false);\n   \n//---\n   \n   this.m_x = x;\n   this.m_y = y;\n   this.x_label = x_axis_label;;\n   this.y_label = y_axis_label;\n   \n   double x_arr[], y_arr[];\n   MatrixExtend::VectorToArray(x, x_arr);\n   MatrixExtend::VectorToArray(y, y_arr);\n   \n   m_curve_type = curve_type;\n   \n   graph.CurveAdd(x_arr, y_arr, ColorToARGB(clr), m_curve_type, label);\n\n   graph.XAxis().Name(x_axis_label);\n   graph.XAxis().NameSize(13);\n   graph.YAxis().Name(y_axis_label);\n   graph.YAxis().NameSize(13);\n   graph.FontSet(m_font_family, 13);\n   graph.CurvePlotAll();\n   graph.Update();\n\n   return(true);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CPlots::AddPlot(vector &v, string label=\"plt\",color clr=clrOrange)\n {\n   double x_arr[], y_arr[];\n   MatrixExtend::VectorToArray(this.m_x, x_arr);\n   MatrixExtend::VectorToArray(v, y_arr);\n \n   if (!graph.CurveAdd(x_arr, y_arr, ColorToARGB(clr), m_curve_type, label))\n    {\n      printf(\"%s failed to add a plot to the existing plot Err =%d\",__FUNCTION__,GetLastError());\n      return false;\n    }\n\n   graph.XAxis().Name(this.x_label);\n   graph.XAxis().NameSize(13);\n   graph.YAxis().Name(this.y_label);\n   graph.YAxis().NameSize(13);\n   graph.FontSet(m_font_family, 13);\n   graph.CurvePlotAll();\n   graph.Update();\n\n   return(true);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Neural Networks/Pattern Nets.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                 Pattern Nets.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| Neural network type for pattern recognition/ can be used to      |\n//| to predict discrete data target variables. They are widely known |\n//| as classification Neural Networks                                |\n//+------------------------------------------------------------------+\n#include <MALE5\\MatrixExtend.mqh>\n#include <MALE5\\preprocessing.mqh>\n\n#ifndef RANDOM_STATE \n #define  RANDOM_STATE 42\n#endif \n \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nenum activation\n  {\n   AF_HARD_SIGMOID_ = AF_HARD_SIGMOID,\n   AF_SIGMOID_ = AF_SIGMOID,\n   AF_SWISH_ = AF_SWISH,\n   AF_SOFTSIGN_ = AF_SOFTSIGN,\n   AF_TANH_ = AF_TANH\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass CPatternNets\n  {\nprivate:\n   \n   vector W_CONFIG;\n   vector W; //Weights vector\n   vector B; //Bias vector \n   activation  A_FX;\n   \nprotected:\n   ulong    inputs;\n   ulong    outputs;\n   ulong    rows;\n   vector   HL_CONFIG;\n   bool     SoftMaxLayer;\n   vector   classes;\n   void     SoftMaxLayerFX(matrix<double> &mat);\n   \npublic:\n                     CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false);\n                    ~CPatternNets(void);\n                    \n                     int  PatternNetFF(vector &in_vector);\n                     vector PatternNetFF(matrix &xmatrix); \n                     \n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPatternNets::CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false)\n  {\n      A_FX = ActivationFx;\n      inputs = xmatrix.Cols();\n      rows = xmatrix.Rows();\n      SoftMaxLayer = SoftMaxLyr;\n      \n//--- Normalize data\n\n      if (rows != yvector.Size())\n        {\n          Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n          return;\n        }\n     \n     classes = MatrixExtend::Unique(yvector);\n     outputs = classes.Size();\n     \n     HL_CONFIG.Copy(HL_NODES);\n      \n     HL_CONFIG.Resize(HL_CONFIG.Size()+1); //Add the output layer\n     HL_CONFIG[HL_CONFIG.Size()-1] = (int)outputs; //Append one node to the output layer\n//---\n     W_CONFIG.Resize(HL_CONFIG.Size());\n     B.Resize((ulong)HL_CONFIG.Sum());\n     \n//--- GENERATE WEIGHTS\n   \n     ulong layer_input = inputs; \n       \n     for (ulong i=0; i<HL_CONFIG.Size(); i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n     W.Resize((ulong)W_CONFIG.Sum());\n     \n     W = MatrixExtend::Random(0.0, 1.0, (int)W.Size(),RANDOM_STATE); //Gen weights\n     B = MatrixExtend::Random(0.0,1.0,(int)B.Size(),RANDOM_STATE); //Gen bias\n      \n//---\n     \n     #ifdef DEBUG_MODE\n       Comment(\n                \"< - - -  P A T T E R N    N E T S  - - - >\\n\",\n                \"HIDDEN LAYERS + OUTPUT \",HL_CONFIG,\"\\n\",   \n                \"INPUTS \",inputs,\" | OUTPUTS \",outputs,\" W CONFIG \",W_CONFIG,\"\\n\",\n                \"activation \",EnumToString(A_FX),\" SoftMaxLayer = \",bool(SoftMaxLayer)\n              );\n              \n       Print(\"WEIGHTS \",W,\"\\nBIAS \",B);\n     #endif \n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPatternNets::~CPatternNets(void)\n  {\n    ZeroMemory(W);\n    ZeroMemory(B);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint CPatternNets::PatternNetFF(vector &in_vector)\n {\n  \n   matrix L_INPUT = {}, L_OUTPUT={}, L_WEIGHTS = {};\n   vector v_weights ={};\n   \n   ulong w_start = 0;             \n   \n   L_INPUT = MatrixExtend::VectorToMatrix(in_vector); \n   \n   vector L_BIAS_VECTOR = {};\n   matrix L_BIAS_MATRIX = {};\n   \n   ulong b_start = 0;\n   \n   for (ulong i=0; i<W_CONFIG.Size(); i++)\n      {         \n         MatrixExtend::Copy(W,v_weights,w_start,ulong(W_CONFIG[i]));\n         \n         L_WEIGHTS = MatrixExtend::VectorToMatrix(v_weights,L_INPUT.Rows());\n         \n         MatrixExtend::Copy(B,L_BIAS_VECTOR,b_start,(ulong)HL_CONFIG[i]);\n         L_BIAS_MATRIX = MatrixExtend::VectorToMatrix(L_BIAS_VECTOR);\n         \n         #ifdef DEBUG_MODE\n           Print(\"--> \",i);\n           Print(\"L_WEIGHTS\\n\",L_WEIGHTS,\"\\nL_INPUT\\n\",L_INPUT,\"\\nL_BIAS\\n\",L_BIAS_MATRIX);\n         #endif \n         \n         L_OUTPUT = L_WEIGHTS.MatMul(L_INPUT);\n\n         L_OUTPUT = L_OUTPUT+L_BIAS_MATRIX; //Add bias\n\n//---\n         \n         if (i==W_CONFIG.Size()-1) //Last layer\n          {\n             if (SoftMaxLayer)  \n              {\n                Print(\"Before softmax\\n\",L_OUTPUT);\n                SoftMaxLayerFX(L_OUTPUT);\n                Print(\"After\\n\",L_OUTPUT);\n              }\n             else\n               L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX));\n          }\n         else\n            L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX));\n            \n//---\n\n         L_INPUT.Copy(L_OUTPUT); //Assign outputs to the inputs\n         w_start += (ulong)W_CONFIG[i]; //New weights copy\n         b_start += (ulong)HL_CONFIG[i];\n         \n      }\n   \n   #ifdef DEBUG_MODE \n     Print(\"--> outputs\\n \",L_OUTPUT);\n   #endif \n   \n   vector v_out = MatrixExtend::MatrixToVector(L_OUTPUT);\n   \n   return((int)classes[v_out.ArgMax()]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nvoid CPatternNets::SoftMaxLayerFX(matrix<double> &mat)\n {\n   vector<double> ret = MatrixExtend::MatrixToVector(mat);\n   \n   ret.Activation(ret, AF_SOFTMAX);\n   \n   mat = MatrixExtend::VectorToMatrix(ret, mat.Cols());\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nvector CPatternNets::PatternNetFF(matrix &xmatrix)\n {\n   vector v(xmatrix.Rows());\n   \n    for (ulong i=0; i<xmatrix.Rows(); i++)\n         v[i] = PatternNetFF(xmatrix.Row(i));      \n   \n   return (v);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Neural Networks/README.md",
    "content": "## Kohonen Maps (Self-Organizing Maps)\n\nThis documentation explains the `CKohonenMaps` class in MQL5, which implements **Kohonen Maps**, also known as **Self-Organizing Maps (SOM)**, for clustering and visualization tasks.\n\n**I. Kohonen Maps Theory:**\n\nKohonen Maps are a type of **artificial neural network** used for unsupervised learning, specifically for **clustering** and **visualization** of high-dimensional data. They work by:\n\n1. **Initializing a grid of neurons:** Each neuron is associated with a weight vector representing its position in the high-dimensional space.\n2. **Iteratively presenting data points:**\n    * For each data point:\n        * Find the **winning neuron** (closest neuron in terms of distance) based on the weight vectors.\n        * Update the weights of the winning neuron and its **neighborhood** towards the data point, with decreasing influence as the distance from the winning neuron increases.\n3. **Convergence:** After a certain number of iterations (epochs), the weight vectors of the neurons become organized in a way that reflects the underlying structure of the data.\n\n**II. CKohonenMaps Class:**\n\nThe `CKohonenMaps` class provides functionalities for implementing Kohonen Maps in MQL5:\n\n**Public Functions:**\n\n* **CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42)** Constructor, allows setting hyperparameters:\n    * `clusters`: Number of clusters (default: 2).\n    * `alpha`: Learning rate (default: 0.01).\n    * `epochs`: Number of training epochs (default: 100).\n    * `random_state`: Random seed for reproducibility (default: 42).\n* `~CKohonenMaps(void)` Destructor.\n* `void fit(const matrix &x)` Trains the model on the provided data (`x`).\n* `int predict(const vector &x)` Predicts the cluster label for a single data point (`x`).\n* `vector predict(const matrix &x)` Predicts cluster labels for multiple data points (`x`).\n\n**Internal Functions:**\n\n* `Euclidean_distance(const vector &v1, const vector &v2)`: Calculates the Euclidean distance between two vectors.\n* `CalcTimeElapsed(double seconds)`: Converts seconds to a human-readable format (not relevant for core functionality).\n\n**III. Additional Notes:**\n\n* The class internally uses the `CPlots` class (not documented here) for potential visualization purposes.\n* The `c_matrix` and `w_matrix` member variables store the cluster assignments and weight matrix, respectively.\n* Choosing the appropriate number of clusters is crucial for the quality of the results.\n\nBy understanding the theoretical foundation and functionalities of the `CKohonenMaps` class, MQL5 users can leverage Kohonen Maps for:\n\n* **Clustering:** Group similar data points together based on their features.\n* **Data visualization:** Project the high-dimensional data onto a lower-dimensional space (e.g., a 2D grid of neurons) for easier visualization, potentially using the `CPlots` class.\n\n\n## Pattern Recognition Neural Network \n\nThis documentation explains the `CPatternNets` class in MQL5, which implements a **feed-forward neural network** for pattern recognition tasks.\n\n**I. Neural Network Theory:**\n\nA feed-forward neural network consists of interconnected layers of **neurons**. Each neuron receives input from the previous layer, applies an **activation function** to transform the signal, and outputs the result to the next layer.\n\n**II. CPatternNets Class:**\n\nThe `CPatternNets` class provides functionalities for training and using a feed-forward neural network for pattern recognition in MQL5:\n\n**Public Functions:**\n\n* `CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false)` Constructor:\n    * `xmatrix`: Input data matrix (rows: samples, columns: features).\n    * `yvector`: Target labels vector (corresponding labels for each sample in `xmatrix`).\n    * `HL_NODES`: Vector specifying the number of neurons in each hidden layer.\n    * `ActivationFx`: Activation function (enum specifying the type of activation function).\n    * `SoftMaxLyr`: Flag indicating whether to use a SoftMax layer in the output (default: False).\n* `~CPatternNets(void)` Destructor.\n* `int PatternNetFF(vector &in_vector)` Performs a forward pass on the network with a single input vector and returns the predicted class label.\n* `vector PatternNetFF(matrix &xmatrix)` Performs a forward pass on the network for all rows in the input matrix and returns a vector of predicted class labels.\n\n**Internal Functions:**\n\n* `SoftMaxLayerFX(matrix<double> &mat)`: Applies the SoftMax function to a matrix (used for the output layer if `SoftMaxLyr` is True).\n\n**III. Class Functionality:**\n\n1. **Initialization:**\n    * The constructor validates data dimensions and parses user-defined hyperparameters.\n    * The network architecture (number of layers and neurons) is determined based on the provided configuration.\n    * Weights (connections between neurons) and biases (individual offsets for each neuron) are randomly initialized.\n\n2. **Forward Pass:**\n    * The provided input vector is fed into the first layer.\n    * Each layer performs the following steps:\n        * Calculates the weighted sum of the previous layer's outputs.\n        * Adds the bias term to the weighted sum.\n        * Applies the chosen activation function to the result.\n    * This process continues through all layers until the final output layer is reached.\n\n3. **SoftMax Layer (Optional)**\n    * If the `SoftMaxLyr` flag is True, the output layer uses the SoftMax function to ensure the output values sum to 1 and represent class probabilities.\n\n4. **Prediction:**\n    * For single-sample prediction (`PatternNetFF(vector &in_vector)`), the class label with the **highest output value** is returned.\n    * For batch prediction (`PatternNetFF(matrix &xmatrix)`), a vector containing the predicted class label for each sample in the input matrix is returned.\n\n**IV. Additional Notes:**\n\n* The class provides several debug statements (disabled by default) to print intermediate calculations for debugging purposes.\n* The code uses helper functions from the `MatrixExtend` class (not documented here) for matrix and vector operations.\n* Choosing the appropriate network architecture, activation function, and learning approach (not implemented in this class) is crucial for optimal performance on specific tasks.\n\nBy understanding the theoretical foundation and functionalities of the `CPatternNets` class, MQL5 users can leverage neural networks for various pattern recognition tasks, including:\n\n* **Classification:** Classifying data points into predefined categories based on their features.\n* **Anomaly detection:** Identifying data points that deviate significantly from the expected patterns.\n* **Feature learning:** Extracting hidden patterns or representations from the data.\n\n\n## Regression Neural Network\n\nThis documentation explains the `CRegressorNets` class in MQL5, which implements a **Multi-Layer Perceptron (MLP)** for regression tasks.\n\n**I. Regression vs. Classification:**\n\n* **Regression:** Predicts continuous output values.\n* **Classification:** Assigns data points to predefined categories.\n\n**II. MLP Neural Network:**\n\nAn MLP is a type of **feed-forward neural network** used for supervised learning tasks like regression. It consists of:\n\n* **Input layer:** Receives the input data.\n* **Hidden layers:** Process and transform the information.\n* **Output layer:** Produces the final prediction (continuous value in regression).\n\n**III. CRegressorNets Class:**\n\nThe `CRegressorNets` class provides functionalities for training and using an MLP for regression in MQL5:\n\n**Public Functions:**\n\n* `CRegressorNets(vector &HL_NODES, activation ActivationFX=AF_RELU_)` Constructor:\n    * `HL_NODES`: Vector specifying the number of neurons in each hidden layer.\n    * `ActivationFX`: Activation function (enum specifying the type of activation function).\n* `~CRegressorNets(void)` Destructor.\n* `void fit(matrix &x, vector &y)` Trains the model on the provided data (`x` - features, `y` - target values).\n* `double predict(vector &x)` Predicts the output value for a single input vector.\n* `vector predict(matrix &x)` Predicts output values for all rows in the input matrix.\n\n**Internal Functions (not directly accessible)**\n\n* `CalcTimeElapsed(double seconds)`: Calculates and returns a string representing the elapsed time in a human-readable format (not relevant for core functionality).\n* `RegressorNetsBackProp(matrix& x, vector &y, uint epochs, double alpha, loss LossFx=LOSS_MSE_, optimizer OPTIMIZER=OPTIMIZER_ADAM)`: Performs backpropagation for training (details not provided but likely involve calculating gradients and updating weights and biases).\n* Optimizer functions (e.g., `AdamOptimizerW`, `AdamOptimizerB`) Implement specific optimization algorithms like Adam for updating weights and biases during training.\n\n**Other Class Members:**\n\n* `mlp_struct mlp`: Stores information about the network architecture (inputs, hidden layers, and outputs).\n* `CTensors*` pointers: Represent tensors holding weights, biases, and other internal calculations (specific implementation likely relies on a custom tensor library).\n* `matrix` variables: Used for calculations during training and may hold temporary data (e.g., `W_MATRIX`, `B_MATRIX`).\n* `vector` variables: Store network configuration details (e.g., `W_CONFIG`, `HL_CONFIG`).\n* `bool isBackProp`: Flag indicating if backpropagation is being performed (private).\n* `matrix` variables: Used for storing intermediate results during backpropagation (e.g., `ACTIVATIONS`, `Partial_Derivatives`).\n\n**IV. Additional Notes:**\n\n* The class provides various activation function options and supports different loss functions (e.g., Mean Squared Error, Mean Absolute Error) for selecting the appropriate evaluation metric during training.\n* The class implements the Adam optimizer, one of several optimization algorithms used for efficient training of neural networks.\n* Detailed implementation of the backpropagation algorithm is not provided but is likely the core functionality for training the network.\n\n\n**Reference**\n* [Data Science and Machine Learning (Part 12): Can Self-Training Neural Networks Help You Outsmart the Stock Market?](https://www.mql5.com/en/articles/12209)\n* [Data Science and Machine Learning — Neural Network (Part 01): Feed Forward Neural Network demystified](https://www.mql5.com/en/articles/11275)\n* [Data Science and Machine Learning — Neural Network (Part 02): Feed forward NN Architectures Design](https://www.mql5.com/en/articles/11334)\n"
  },
  {
    "path": "Neural Networks/Regressor Nets.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                neural_nn_lib.mqh |\n//|                                    Copyright 2022, Omega Joctan. |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Omega Joctan.\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n//|  Regressor Neural Networks | Neural Networks for solving         |\n//|  regression problems in contrast to classification problems,     |\n//|  here we deal with continuous variables                          |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\preprocessing.mqh>;\n#include <MALE5\\MatrixExtend.mqh>;\n#include <MALE5\\Metrics.mqh>\n#include <MALE5\\Tensors.mqh>\n#include <MALE5\\cross_validation.mqh>\n#include \"optimizers.mqh\"\n\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nenum activation\n  {\n   AF_ELU_ = AF_ELU,\n   AF_EXP_ = AF_EXP,\n   AF_GELU_ = AF_GELU,\n   AF_LINEAR_ = AF_LINEAR,\n   AF_LRELU_ = AF_LRELU,\n   AF_RELU_ = AF_RELU,\n   AF_SELU_ = AF_SELU,\n   AF_TRELU_ = AF_TRELU,\n   AF_SOFTPLUS_ = AF_SOFTPLUS\n  };\n  \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nenum loss\n  {\n    LOSS_MSE_ = LOSS_MSE,  // Mean Squared Error\n    LOSS_MAE_ = LOSS_MAE,  // Mean Absolute Error\n    LOSS_MSLE_ = LOSS_MSLE,  // Mean Squared Logarithmic Error\n    LOSS_POISSON_ = LOSS_POISSON  // Poisson Loss\n  };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nstruct backprop //This structure returns the loss information obtained from the backpropagation function\n  {\n    vector training_loss,\n           validation_loss;\n           \n           void Init(ulong epochs)\n            {\n              training_loss.Resize(epochs);\n              validation_loss.Resize(epochs);\n            }\n  };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nstruct mlp_struct //multi layer perceptron information structure\n {\n   ulong inputs;\n   ulong hidden_layers;\n   ulong outputs;\n };\n  \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass CRegressorNets\n  {   \n   mlp_struct        mlp;\n   \n   CTensors          *Weights_tensor; //Weight Tensor\n   CTensors          *Bias_tensor;\n   CTensors          *Input_tensor;\n   CTensors          *Output_tensor;\n   \nprotected:\n   activation        A_FX;\n   loss              m_loss_function;\n   bool              trained;\n\n   string            ConvertTime(double seconds);\n\n//-- for backpropn\n\n   vector            W_CONFIG;\n   vector            HL_CONFIG; \n   \n   bool              isBackProp; \n   matrix<double>    ACTIVATIONS;\n   matrix<double>    Partial_Derivatives; \n   int               m_random_state;\n   \nprivate: \n   \n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual backprop  backpropagation(const matrix& x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   \npublic:\n\n                              CRegressorNets(vector &HL_NODES, activation AF_=AF_RELU_, loss m_loss_function=LOSS_MSE_, int random_state=42);\n                             ~CRegressorNets(void);\n\n   virtual void              fit(const matrix &x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual void              fit(const matrix &x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual void              fit(const matrix &x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual void              fit(const matrix &x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual void              fit(const matrix &x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   virtual void              fit(const matrix &x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);\n   \n   virtual double            predict(const vector &x);\n   virtual vector            predict(const matrix &x);\n  };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRegressorNets::CRegressorNets(vector &HL_NODES, activation AF_=AF_RELU_, loss LOSS_=LOSS_MSE_, int random_state=42)\n :A_FX(AF_),\n  m_loss_function(LOSS_),\n  isBackProp(false),\n  m_random_state(random_state)\n  {   \n     HL_CONFIG.Copy(HL_NODES);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRegressorNets::~CRegressorNets(void)\n  {\n   if (CheckPointer(this.Weights_tensor) != POINTER_INVALID)  delete(this.Weights_tensor);\n   if (CheckPointer(this.Bias_tensor) != POINTER_INVALID)  delete(this.Bias_tensor);\n   if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n   if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor);\n   \n   isBackProp = false; \n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CRegressorNets::predict(const vector &x)\n  {   \n  \n  if (!trained)\n   {\n     printf(\"%s Train the model first before using it to make predictions | call the fit function first\",__FUNCTION__);\n     return 0;\n   }\n   \n   matrix L_INPUT = MatrixExtend::VectorToMatrix(x); \n   matrix L_OUTPUT ={};\n    \n   for(ulong i=0; i<mlp.hidden_layers; i++)\n     {      \n      if (isBackProp) //if we are on backpropagation store the inputs to be used for finding derivatives \n        this.Input_tensor.Add(L_INPUT, i);  \n\n      L_OUTPUT = this.Weights_tensor.Get(i).MatMul(L_INPUT) + this.Bias_tensor.Get(i); //Weights x INputs + Bias \n\n      L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX)); //Activation\n      \n      L_INPUT = L_OUTPUT; //Next layer inputs = previous layer outputs\n      \n      if (isBackProp)  this.Output_tensor.Add(L_OUTPUT, i); //Add bias //if we are on backpropagation store the outputs to be used for finding derivatives \n     }\n   \n   return(L_OUTPUT[0][0]);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CRegressorNets::predict(const matrix &x)\n {\n  ulong size = x.Rows();\n  \n  vector v(size);\n   if (x.Cols() != mlp.inputs)\n    {\n       Print(\"Cen't pass this matrix to a MLP it doesn't have the same number of columns as the inputs given primarily\");\n       return (v); \n    }\n\n   for (ulong i=0; i<size; i++)\n     v[i] = predict(x.Row(i));    \n    \n   return (v);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nstring CRegressorNets::ConvertTime(double seconds)\n{\n    string time_str = \"\";\n    uint minutes = 0, hours = 0;\n\n    if (seconds >= 60)\n    {\n        minutes = (uint)(seconds / 60.0) ;\n        seconds = fmod(seconds, 1.0) * 60;\n        time_str = StringFormat(\"%d Minutes and %.3f Seconds\", minutes, seconds);\n    }\n    \n    if (minutes >= 60)\n    {\n        hours = (uint)(minutes / 60.0);\n        minutes = minutes % 60;\n        time_str = StringFormat(\"%d Hours and %d Minutes\", hours, minutes);\n    }\n\n    if (time_str == \"\")\n    {\n        time_str = StringFormat(\"%.3f Seconds\", seconds);\n    }\n\n    return time_str;\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerSGD optimizer_weights = optimizer;\n   OptimizerSGD optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n  \n  \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerAdaDelta optimizer_weights = optimizer;\n   OptimizerAdaDelta optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n   \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerAdaGrad optimizer_weights = optimizer;\n   OptimizerAdaGrad optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n   \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerAdam optimizer_weights = optimizer;\n   OptimizerAdam optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n   \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerNadam optimizer_weights = optimizer;\n   OptimizerNadam optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n   \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbackprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n   isBackProp = true;\n   \n//---\n\n   backprop backprop_struct;\n   backprop_struct.Init(epochs);\n   \n   ulong rows = x.Rows();\n   \n   mlp.inputs = x.Cols();\n   mlp.outputs = 1;\n   \n//---\n\n   vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers\n  \n   HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);\n   mlp.hidden_layers = HL_CONFIG.Size();\n   W_CONFIG.Resize(HL_CONFIG.Size());\n     \n//---\n\n   if (y.Size() != rows)\n     {\n        Print(__FUNCTION__,\" FATAL | Number of rows in the x matrix is not the same the y vector size \");\n        return backprop_struct;\n     }\n     \n     \n     matrix W, B;\n     \n//--- GENERATE WEIGHTS\n    \n     this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Input_tensor = new CTensors((uint)mlp.hidden_layers);\n     this.Output_tensor = new CTensors((uint)mlp.hidden_layers);\n     \n     ulong layer_input = mlp.inputs; \n     \n     for (ulong i=0; i<mlp.hidden_layers; i++)\n       {\n          W_CONFIG[i] = layer_input*HL_CONFIG[i];\n          \n          W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);\n          \n          W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Weights_tensor.Add(W, i);\n          \n          B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);\n          \n          B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot\n          this.Bias_tensor.Add(B, i);\n          \n          layer_input = (ulong)HL_CONFIG[i];\n       }\n     \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Comment(\"<-------------------  R E G R E S S O R   N E T S  ------------------------->\\n\",\n            \"HL_CONFIG \",HL_CONFIG,\" TOTAL HL(S) \",mlp.hidden_layers,\"\\n\",\n            \"W_CONFIG \",W_CONFIG,\" ACTIVATION \",EnumToString(A_FX),\"\\n\",\n            \"NN INPUTS \",mlp.inputs,\" OUTPUT \",mlp.outputs\n           );\n\n//--- Optimizer\n      \n   OptimizerRMSprop optimizer_weights = optimizer;\n   OptimizerRMSprop optimizer_bias = optimizer;\n   \n   if (batch_size>0)\n    {\n      OptimizerMinBGD optimizer_weights;\n      OptimizerMinBGD optimizer_bias;\n    }\n     \n//--- Cross validation\n\n    CCrossValidation cross_validation;      \n    CTensors *cv_tensor;\n    matrix validation_data = MatrixExtend::concatenate(x, y);\n    matrix validation_x;\n    vector validation_y;\n    \n    cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected\n    \n//---\n\n    matrix DELTA = {};\n    double actual=0, pred=0;\n    \n    matrix temp_inputs ={};\n    \n    matrix dB = {}; //Bias Derivatives\n    matrix dW = {}; //Weight Derivatives\n    \n   \n    for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)\n      {        \n        double epoch_start = GetTickCount(); \n\n        uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));\n        \n        vector batch_loss(num_batches), \n               batch_accuracy(num_batches);\n                       \n         vector actual_v(1), pred_v(1), LossGradient = {};\n         \n         if (batch_size==0) //Stochastic Gradient Descent\n          {\n           for (ulong iter=0; iter<rows; iter++) //iterate through all data points\n             {\n               pred = predict(x.Row(iter));\n               actual = y[iter];\n               \n               pred_v[0] = pred; \n               actual_v[0] = actual; \n   //---\n                \n                DELTA.Resize(mlp.outputs,1);\n                \n                for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                  {    \n                     Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                     temp_inputs = this.Input_tensor.Get(int(layer));\n                     \n                     Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                     \n                     if (mlp.hidden_layers-1 == layer) //Last layer\n                      {                     \n                        LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                        \n                        DELTA.Col(LossGradient, 0);\n                      }\n                      \n                     else\n                      {\n                        W = this.Weights_tensor.Get(layer+1);\n                        \n                        DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                      }\n                    \n                    //-- Observation | DeLTA matrix is same size as the bias matrix\n                    \n                    W = this.Weights_tensor.Get(layer);\n                    B = this.Bias_tensor.Get(layer);\n                  \n                   //--- Derivatives wrt weights and bias\n                  \n                    dB = DELTA;\n                    dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                    \n                   //--- Weights updates\n                    \n                    optimizer_weights.update(W, dW);\n                    optimizer_bias.update(B, dB);\n                    \n                    this.Weights_tensor.Add(W, layer);\n                    this.Bias_tensor.Add(B, layer);\n                  }\n             }\n         }\n        else //Batch Gradient Descent\n          {\n               \n            for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))\n               {\n                  matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);\n                  vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);\n                  \n                  rows = batch_x.Rows();              \n                  \n                    for (ulong iter=0; iter<rows ; iter++) //iterate through all data points\n                      {\n                        pred_v[0] = predict(batch_x.Row(iter));\n                        actual_v[0] = y[iter];\n                        \n            //---\n                        \n                      DELTA.Resize(mlp.outputs,1);\n                      \n                      for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer\n                        {    \n                           Partial_Derivatives = this.Output_tensor.Get(int(layer));\n                           temp_inputs = this.Input_tensor.Get(int(layer));\n                           \n                           Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));\n                           \n                           if (mlp.hidden_layers-1 == layer) //Last layer\n                            {                     \n                              LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));\n                              \n                              DELTA.Col(LossGradient, 0);\n                            }\n                            \n                           else\n                            {\n                              W = this.Weights_tensor.Get(layer+1);\n                              \n                              DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;\n                            }\n                          \n                          //-- Observation | DeLTA matrix is same size as the bias matrix\n                          \n                          W = this.Weights_tensor.Get(layer);\n                          B = this.Bias_tensor.Get(layer);\n                        \n                         //--- Derivatives wrt weights and bias\n                        \n                          dB = DELTA;\n                          dW = DELTA.MatMul(temp_inputs.Transpose());                   \n                          \n                         //--- Weights updates\n                          \n                          optimizer_weights.update(W, dW);\n                          optimizer_bias.update(B, dB);\n                          \n                          this.Weights_tensor.Add(W, layer);\n                          this.Bias_tensor.Add(B, layer);\n                        }\n                    }\n                 \n                 pred_v = predict(batch_x);\n                 \n                 batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));\n                 batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan\n                 \n                 batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);\n                 \n                 if (show_batch_progress)\n                  printf(\"----> batch[%d/%d] batch-loss %.5f accuracy %.3f\",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);  \n              }\n          }\n          \n//--- End of an epoch\n      \n        vector validation_loss(cv_tensor.SIZE);\n        vector validation_acc(cv_tensor.SIZE);\n        for (ulong i=0; i<cv_tensor.SIZE; i++)\n          {\n            validation_data = cv_tensor.Get(i);\n            MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);\n            \n            vector val_preds = this.predict(validation_x);;\n            \n            validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));\n            validation_acc[i] = Metrics::r_squared(validation_y, val_preds);\n          }\n                  \n        pred_v = this.predict(x);\n        \n        if (batch_size==0)\n          {      \n              backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n        else\n          {\n              backprop_struct.training_loss[epoch] = batch_loss.Mean();\n              backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n              backprop_struct.validation_loss[epoch] = validation_loss.Mean();\n              backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan\n          }\n          \n        double epoch_stop = GetTickCount();  \n        printf(\"--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s \",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));\n     }\n     \n   isBackProp = false;\n   \n  if (CheckPointer(this.Input_tensor) != POINTER_INVALID)  delete(this.Input_tensor);\n  if (CheckPointer(this.Output_tensor) != POINTER_INVALID)  delete(this.Output_tensor); \n  if (CheckPointer(optimizer)!=POINTER_INVALID)  \n    delete optimizer;\n    \n   return backprop_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-SGD log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-AdaDelta log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-AdaGrad log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-Adam log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-Nadam log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRegressorNets::fit(const matrix &x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)\n {\n  trained = true; //The fit method has been called\n  \n  vector epochs_vector(epochs);  for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;\n  \n  backprop backprop_struct;\n  \n  backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation\n  \n\n  CPlots plt;\n    \n  backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling\n  plt.Plot(\"Loss vs Epochs\",epochs_vector,backprop_struct.training_loss,\"epochs\",\"optimizer-RMSProp log10(loss)\",\"training-loss\",CURVE_LINES);\n  backprop_struct.validation_loss = log10(backprop_struct.validation_loss);\n  plt.AddPlot(backprop_struct.validation_loss,\"validation-loss\",clrRed);\n  \n   while (MessageBox(\"Close or Cancel Loss Vs Epoch plot to proceed\",\"Training progress\",MB_OK)<0)\n    Sleep(1);\n\n  isBackProp = false;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Neural Networks/initializers.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                initiallizers.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n#include <MALE5\\MatrixExtend.mqh>\n\nenum enum_weight_initializers\n {\n   WEIGHT_INTIALIZER_XAVIER, //Xavier weights\n   WEIGHT_INTIALIZER_HE, //He weights \n   WEIGHT_INTIALIZER_RANDOM //Random weights \n };\n\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\nclass CWeightsInitializers\n  {\nprotected:\n\n   static double random()  //Generates a random float between 0 (inclusive) and 1 (exclusive)\n     {              \n       return 0 + double((MathRand() / 32767.0) * (0.9 - 0));\n     }\n     \n   static double uniform(double low, double high)\n     {\n       return low + (high - low) * random();\n     }\n     \n   static matrix uniform(double low, double high, ulong rows, ulong cols)\n    {\n      matrix return_matrix(rows, cols);\n      for (ulong i=0; i<rows; i++)\n        for (ulong j=0; j<cols; j++)\n           return_matrix[i][j] = uniform(low, high);\n      \n      return return_matrix;\n    }\n   \n   static vector uniform(double low, double high, ulong size)\n    {\n      vector v(size);\n      for (ulong i=0; i<size; i++)\n        v[i] = uniform(low, high);\n      \n      return v;\n    }\n   \npublic:\n                     CWeightsInitializers(void);\n                    ~CWeightsInitializers(void);\n                    \n                    static matrix Xavier(const ulong inputs, const ulong outputs); //Xavier or Glorot initialization\n                    static matrix He(const ulong inputs, const ulong outputs);\n                    static matrix Random(const ulong inputs, const ulong outputs);\n                    static matrix Initializer(const ulong inputs, const ulong outputs, enum_weight_initializers WEIGHT_INIT=WEIGHT_INTIALIZER_XAVIER);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CWeightsInitializers::Xavier(const ulong inputs,const ulong outputs)\n {\n   #ifdef RANDOM_STATE\n     MathSrand(RANDOM_STATE);\n   #endif \n   \n   double limit = sqrt(6/(inputs+outputs+DBL_EPSILON));\n   return uniform(-limit, limit, inputs, outputs);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CWeightsInitializers::He(const ulong inputs, const ulong outputs)\n {\n   #ifdef RANDOM_STATE\n     MathSrand(RANDOM_STATE);\n   #endif \n   \n   double limit = sqrt(2 / (inputs+DBL_EPSILON));\n   \n   matrix W(inputs, outputs);\n   for (ulong i=0; i<outputs; i++)\n    {\n      vector v = uniform(-limit, limit, inputs);\n      W.Col(v, i);\n    }\n   \n   return uniform(-limit, limit, inputs, outputs);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CWeightsInitializers::Random(const ulong inputs,const ulong outputs)\n {\n   #ifdef RANDOM_STATE\n     MathSrand(RANDOM_STATE);\n   #endif \n   \n   matrix W(inputs, outputs);\n   for (ulong i=0; i<inputs; i++)\n     for (ulong j=0; j<outputs; j++)\n       W[i][j] = random();\n       \n   return W;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CWeightsInitializers::Initializer(const ulong inputs,const ulong outputs,enum_weight_initializers WEIGHT_INIT=WEIGHT_INTIALIZER_XAVIER)\n {\n   matrix W = {};\n   \n   switch(WEIGHT_INIT)\n     {\n      case  WEIGHT_INTIALIZER_HE:\n        W = He(inputs, outputs);\n        break;\n      case  WEIGHT_INTIALIZER_RANDOM:\n        W = Random(inputs, outputs);\n        break;\n      case  WEIGHT_INTIALIZER_XAVIER:\n        W = Xavier(inputs, outputs);\n        break;\n     }\n     \n   return W;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n"
  },
  {
    "path": "Neural Networks/kohonen maps.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                 kohonen maps.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n#include <MALE5\\MatrixExtend.mqh>\n#include <MALE5\\Tensors.mqh>\n#include <MALE5\\preprocessing.mqh>\n#include <MALE5\\MqPlotLib\\plots.mqh>\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CKohonenMaps\n  {\n   protected:\n     C2DTensor *cluster_tensor;\n     \n     CPlots   plt;\n      \n      double  Euclidean_distance(const vector &v1, const vector &v2);\n      string  CalcTimeElapsed(double seconds);\n      \n      matrix     c_matrix; //Clusters\n      matrix     w_matrix; //Weights matrix\n      vector     w_vector; //weights vector\n      matrix     o_matrix; //Output layer matrix\n      \n      uint m_clusters;\n      double m_alpha;\n      uint m_epochs; \n      int m_random_state;\n      ulong n, m;\n      \n   public:\n                  CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42);\n                 ~CKohonenMaps(void);\n                 \n                  void fit(const matrix &x);\n                  int predict(const vector &x);\n                  vector predict(const matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCKohonenMaps::CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42)\n :m(clusters),\n m_alpha(alpha),\n m_epochs(epochs),\n m_random_state(random_state),\n m_clusters(clusters)\n {   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CKohonenMaps::fit(const matrix &x)\n {\n   n = (uint)x.Cols(); //number of features \n   ulong rows = x.Rows();\n   \n   cluster_tensor = new C2DTensor();\n   cluster_tensor.Init((int)m);\n   \n   w_matrix =MatrixExtend::Random(0.0, 1.0, n, m, m_random_state); \n   \n   #ifdef DEBUG_MODE\n      Print(\"w x\\n\",w_matrix,\"\\nMatrix\\n\",x);\n   #endif \n   \n   vector D(m); //Euclidean distance btn clusters\n   \n   \n   for (uint epoch=0; epoch<m_epochs; epoch++)\n    {\n    \n      double epoch_start = GetMicrosecondCount()/(double)1e6, epoch_stop=0; \n      \n      for (ulong i=0; i<rows; i++)\n       {\n         for (ulong j=0; j<m; j++)\n           {\n             D[j] = Euclidean_distance(x.Row(i),w_matrix.Col(j));\n           }\n         \n         #ifdef DEBUG_MODE  \n            //Print(\"Euc distance \",D,\" Winning cluster \",D.ArgMin());\n         #endif \n         \n   //--- weights update\n         \n         ulong min = D.ArgMin();\n         \n         if (epoch == m_epochs-1) //last iteration\n            cluster_tensor[(int)min].Vector = x.Row(i);; \n\n          \n         vector w_new =  w_matrix.Col(min) + (m_alpha * (x.Row(i) - w_matrix.Col(min)));\n         \n         w_matrix.Col(w_new, min);\n       }\n  \n      epoch_stop =GetMicrosecondCount()/(double)1e6;    \n      \n      printf(\"Epoch [%d/%d] | %sElapsed \",epoch+1,m_epochs, CalcTimeElapsed(epoch_stop-epoch_start));\n      \n    }  //end of the training\n\n//---\n\n  #ifdef DEBUG_MODE\n      Print(\"\\nNew weights\\n\",w_matrix);\n  #endif \n\n//---\n   \n\n   vector v;  \n   matrix plotmatrix(rows, m); \n   \n     for (uint i=0; i<this.cluster_tensor.Size(); i++)\n       {\n          v = this.cluster_tensor[i].Vector;\n                    \n          plotmatrix.Col(v, i);\n       }   \n    \n    vector x_axis(plotmatrix.Rows());    for (ulong i=0; i<x_axis.Size(); i++) x_axis[i] = (int)i+1;\n    \n    CColorGenerator clr;\n    \n    plt.Plot(\"kom\", x_axis, plotmatrix.Col(0), \"map\", \"clusters\",\"cluster\"+string(1),CURVE_POINTS,clr.Next()); //plot the first cluster\n    for (ulong i=1; i<plotmatrix.Cols(); i++) //start at the second column in the matrix | the second cluster\n      {\n        plt.AddPlot(plotmatrix.Col(i), \"cluster\"+string(i+1),clr.Next()); //Add the rest of clusters to the existing plot \n      }\n\n//---\n   \n   if (MQLInfoInteger(MQL_DEBUG))\n    {\n     Print(\"\\nclusters\");\n     cluster_tensor.Print_();\n   }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCKohonenMaps::~CKohonenMaps(void)\n {\n   ZeroMemory(c_matrix); \n   ZeroMemory(w_matrix); \n   ZeroMemory(w_vector); \n   ZeroMemory(o_matrix); \n   delete (cluster_tensor);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\ndouble CKohonenMaps:: Euclidean_distance(const vector &v1, const vector &v2)\n  {\n   double dist = 0;\n\n   if(v1.Size() != v2.Size())\n      Print(__FUNCTION__, \" v1 and v2 not matching in size\");\n   else\n     {\n      double c = 0;\n      for(ulong i=0; i<v1.Size(); i++)\n         c += MathPow(v1[i] - v2[i], 2);\n\n      dist = MathSqrt(c);\n     }\n\n   return(dist);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint CKohonenMaps::predict(const vector &v)\n {  \n  if (n != v.Size())\n   {\n     printf(\"%s Can't predict the cluster | the input vector size is not the same as the trained matrix cols\",__FUNCTION__);\n     return(-1);\n   }\n   \n   vector D(m); //Euclidean distance btn clusters\n   \n   for (ulong j=0; j<m; j++)\n       D[j] = Euclidean_distance(v, w_matrix.Col(j));\n\n   return((int)D.ArgMin());\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nvector CKohonenMaps::predict(const matrix &x)\n {   \n   vector v(x.Rows());\n   \n   for (ulong i=0; i<x.Rows(); i++)\n      v[i] = predict(x.Row(i));\n      \n    return(v);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nstring CKohonenMaps::CalcTimeElapsed(double seconds)\n {\n  string time_str = \"\";\n  \n  uint minutes=0, hours=0;\n  \n   if (seconds >= 60)\n     time_str = StringFormat(\"%d Minutes and %.3f Seconds \",minutes=(int)round(seconds/60.0), ((int)seconds % 60));     \n   if (minutes >= 60)\n     time_str = StringFormat(\"%d Hours %d Minutes and %.3f Seconds \",hours=(int)round(minutes/60.0), minutes, ((int)seconds % 60));\n   else\n     time_str = StringFormat(\"%.3f Seconds \",seconds);\n     \n   return time_str;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Neural Networks/optimizers.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                   optimizers.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n//|Class containing optimizers for updating neural network parameters|\n//+------------------------------------------------------------------+\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//| Adam (Adaptive Moment Estimation): Adam is an adaptive learning  |\n//| rate optimizer that maintains learning rates for each parameter  |\n//| and adapts them based on the first and second moments of the     |\n//| gradients. It's known for its fast convergence and robustness to |\n//| noisy gradients                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n/*enum Optimizer {\n    OPTIMIZER_SGD, //Stochastic Gradient Descent\n    OPTIMIZER_MiniBatchGD, //Mini-Batch Gradiend Descent\n    OPTIMIZER_Adam, //Adaptive Moment Estimation\n    OPTIMIZER_RMSprop, //Root Mean Square Propagation\n    OPTIMIZER_Adagrad, //Adaptive Gradient Descent \n    OPTIMIZER_Adadelta, //Adadelta\n    OPTIMIZER_Nadam //Nesterov-accelerated Adaptive Moment Estimation\n};\n*/\n\nclass OptimizerAdam\n  {\nprotected:\n   int time_step;\n   double m_learning_rate;\n   double m_beta1;\n   double m_beta2;\n   double m_epsilon;\n   \n   matrix moment; //first moment estimate\n   matrix cache; //second moment estimate\n   \npublic:\n                     OptimizerAdam(double learning_rate=0.01, double beta1=0.9, double beta2=0.999, double epsilon=1e-8);\n                    ~OptimizerAdam(void);\n                    \n                    virtual void update(matrix &parameters, matrix &gradients);\n  };\n//+------------------------------------------------------------------+\n//|  Initializes the Adam optimizer with hyperparameters.            |   \n//|                                                                  |\n//|  learning_rate: Step size for parameter updates                  |\n//|  beta1: Decay rate for the first moment estimate                 |\n//|     (moving average of gradients).                               |\n//|  beta2: Decay rate for the second moment estimate                |\n//|     (moving average of squared gradients).                       |\n//|  epsilon: Small value for numerical stability.                   |\n//+------------------------------------------------------------------+\nOptimizerAdam::OptimizerAdam(double learning_rate=0.010000, double beta1=0.9, double beta2=0.999, double epsilon=1e-8):\n time_step(0),\n m_learning_rate(learning_rate),\n m_beta1(beta1),\n m_beta2(beta2),\n m_epsilon(epsilon)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerAdam::~OptimizerAdam(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerAdam::update(matrix &parameters,matrix &gradients)\n {\n    // Initialize moment and cache matrices if not already initialized\n    if (moment.Rows() != parameters.Rows() || moment.Cols() != parameters.Cols())\n    {\n        moment.Resize(parameters.Rows(), parameters.Cols());\n        moment.Fill(0.0);\n    }\n\n    if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())\n    {\n        cache.Resize(parameters.Rows(), parameters.Cols());\n        cache.Fill(0.0);\n    }\n\n   \n    this.time_step++; \n    \n    this.moment = this.m_beta1 * this.moment + (1 -  this.m_beta1) * gradients;\n    \n    this.cache = this.m_beta2 * this.cache + (1 -  this.m_beta2) * MathPow(gradients, 2);\n\n//--- Bias correction\n\n    matrix moment_hat = this.moment / (1 - MathPow(this.m_beta1, this.time_step));\n    \n    matrix cache_hat = this.cache / (1 - MathPow(this.m_beta2, this.time_step));\n    \n    parameters -= (this.m_learning_rate * moment_hat) / (MathPow(cache_hat, 0.5) + this.m_epsilon);\n }\n \n \n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//| Stochastic Gradient Descent (SGD):                               |\n//|                                                                  |\n//| This is the simplest optimizer                                   |\n//| It updates the parameters using the gradients of the loss        |\n//| function computed on individual training samples or mini-batches.|\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\nclass OptimizerSGD\n  {\nprotected:\n   double m_learning_rate;\n   \npublic:\n                     OptimizerSGD(double learning_rate=0.01);\n                    ~OptimizerSGD(void);\n                     \n                    virtual void update(matrix &parameters, matrix &gradients);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerSGD::OptimizerSGD(double learning_rate=0.01):\n m_learning_rate(learning_rate)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerSGD::~OptimizerSGD(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerSGD::update(matrix &parameters, matrix &gradients)\n {\n    parameters -= this.m_learning_rate * gradients;\n }\n \n \n//+------------------------------------------------------------------+\n//|  Batch Gradient Descent (BGD): This optimizer computes the       |\n//|  gradients of the loss function on the entire training dataset   |\n//|  and updates the parameters accordingly. It can be slow and      |\n//|  memory-intensive for large datasets but tends to provide a      |\n//|  stable convergence.                                             |\n//+------------------------------------------------------------------+\n\n\nclass OptimizerMinBGD: public OptimizerSGD\n  {\npublic:\n                     OptimizerMinBGD(double learning_rate=0.01);\n                    ~OptimizerMinBGD(void);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerMinBGD::OptimizerMinBGD(double learning_rate=0.010000): OptimizerSGD(learning_rate)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerMinBGD::~OptimizerMinBGD(void)\n {\n \n }\n \n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//| RMSprop (Root Mean Square Propagation): RMSprop is similar to    |\n//| Adam but only uses the first moment of the gradients to adapt the|\n//| moment learning rates. It's effective in handling non-stationary      |\n//| objectives and can converge quickly for many problems.           |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\nclass OptimizerRMSprop\n  {\nprotected:\n   double m_learning_rate;\n   double m_decay_rate;\n   double m_epsilon;\n   \n   matrix<double> cache;\n   \n   //Dividing double/matrix causes compilation error | this is the fix to the issue\n   matrix divide(const double numerator, const matrix &denominator)\n    {\n      matrix res = denominator;\n      \n      for (ulong i=0; i<denominator.Rows(); i++)\n        res.Row(numerator / denominator.Row(i), i);\n     return res;\n    }\n    \npublic:\n                     OptimizerRMSprop(double learning_rate=0.01, double decay_rate=0.9, double epsilon=1e-8);\n                    ~OptimizerRMSprop(void);\n                    \n                    virtual void update(matrix& parameters, matrix& gradients);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerRMSprop::OptimizerRMSprop(double learning_rate=0.01, double decay_rate=0.9, double epsilon=1e-8):\n m_learning_rate(learning_rate),\n m_decay_rate(decay_rate),\n m_epsilon(epsilon)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerRMSprop::~OptimizerRMSprop(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerRMSprop::update(matrix &parameters,matrix &gradients)\n {\n \n   if (cache.Rows()!=parameters.Rows() || cache.Cols()!=parameters.Cols())\n    {\n     cache.Init(parameters.Rows(), parameters.Cols());\n     cache.Fill(0.0);\n    }\n     \n//---\n    \n    cache += m_decay_rate * cache + (1 - m_decay_rate) * MathPow(gradients, 2);\n    parameters -= divide(m_learning_rate, cache + m_epsilon) * gradients;\n }\n\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//| Adagrad (Adaptive Gradient Algorithm):                           |\n//|                                                                  |\n//| Adagrad adapts the learning rates for each parameter based on the|\n//| historical gradients. It performs larger updates for infrequent  |\n//| parameters and smaller updates for frequent parameters, making   |\n//| it suitable for sparse data.                                     |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass OptimizerAdaGrad\n  {\nprotected:\n double m_learning_rate;\n double m_epsilon;\n matrix cache;\n \n \n   //Dividing double/matrix causes compilation error | this is the fix to the issue\n   matrix divide(const double numerator, const matrix &denominator)\n    {\n      matrix res = denominator;\n      \n      for (ulong i=0; i<denominator.Rows(); i++)\n        res.Row(numerator / denominator.Row(i), i);\n     return res;\n    }\n \npublic:\n                     OptimizerAdaGrad(double learning_rate=0.01, double epsilon=1e-8);\n                    ~OptimizerAdaGrad(void);\n                    \n                    virtual void update(matrix &parameters, matrix &gradients);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerAdaGrad::OptimizerAdaGrad(double learning_rate=0.01,double epsilon=1e-8):\n m_learning_rate(learning_rate),\n m_epsilon(epsilon)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerAdaGrad::~OptimizerAdaGrad(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerAdaGrad::update(matrix &parameters,matrix &gradients)\n {\n   if (cache.Rows()!=parameters.Rows() || cache.Cols()!=parameters.Cols())\n    {\n     cache.Resize(parameters.Rows(), parameters.Cols());\n     cache.Fill(0.0);\n    }\n     \n//--- \n   \n    cache += MathPow(gradients, 2);\n    parameters -= divide(this.m_learning_rate,  MathSqrt(cache + this.m_epsilon)) * gradients;\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//| Adadelta:                                                        |      \n//|                                                                  |\n//| Adadelta is an extension of Adagrad that aims to address         |\n//| its tendency to decrease the learning rate over time. It uses a  |\n//| more sophisticated update rule that accounts for the accumulated |\n//| gradients over a window of time.                                 |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\nclass OptimizerAdaDelta\n  {\nprotected:\n   double m_decay_rate;\n   double m_epsilon, m_gamma, lr;\n   matrix cache; //Exponential moving average of squared gradients\n   \npublic:\n                     OptimizerAdaDelta(double learning_rate=0.01, double decay_rate=0.95, double gamma=0.9, double epsilon=1e-8);\n                    ~OptimizerAdaDelta(void);\n                    \n                     virtual void update(matrix &parameters, matrix &gradients);\n  };\n//+------------------------------------------------------------------+\n//|   decay_rate: Decay rate for the EMA of squared deltas           |\n//|   epsilons: Smoothing term to avoid division by zero             |\n//|   gamma: Momentum coefficient (hyperparameter)                   |\n//+------------------------------------------------------------------+\nOptimizerAdaDelta::OptimizerAdaDelta(double learning_rate=0.01, double decay_rate=0.95, double gamma=0.9, double epsilon=1e-8):\n  m_decay_rate(decay_rate),\n  m_epsilon(epsilon),\n  m_gamma(gamma),\n  lr(learning_rate)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerAdaDelta::~OptimizerAdaDelta(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerAdaDelta::update(matrix &parameters, matrix &gradients)\n {\n    // Initialize moment and cache matrices if not already initialized\n    if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())\n    {\n        cache.Resize(parameters.Rows(), parameters.Cols());\n        cache.Fill(0.0);\n    }\n\n//---\n   \n   this.cache = m_decay_rate * this.cache + (1 - m_decay_rate) * MathPow(gradients, 2);\n   \n   matrix delta = lr * sqrt(this.cache + m_epsilon) / sqrt(pow(gradients, 2) + m_epsilon); //Adaptive learning rate\n   \n   matrix momentum_term = this.m_gamma * parameters + (1 - this.m_gamma) * gradients;\n   \n   parameters -= delta * momentum_term;\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//| Nadam (Nesterov-accelerated Adaptive Moment Estimation):         |\n//|                                                                  |\n//| Nadam combines Nesterov momentum with Adam optimizer, which      |\n//| results in faster convergence and better generalization.         |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\n\nclass OptimizerNadam: protected OptimizerAdam\n  {\nprotected:\n   double m_gamma;\n   \n    \npublic:\n                     OptimizerNadam(double learning_rate=0.01, double beta1=0.9, double beta2=0.999, double gamma=0.9, double epsilon=1e-8);\n                    ~OptimizerNadam(void);\n                    \n                    virtual void update(matrix &parameters, matrix &gradients);\n  };\n//+------------------------------------------------------------------+\n//|  Initializes the Adam optimizer with hyperparameters.            |   \n//|                                                                  |\n//|  learning_rate: Step size for parameter updates                  |\n//|  beta1: Decay rate for the first moment estimate                 |\n//|     (moving average of gradients).                               |\n//|  beta2: Decay rate for the second moment estimate                |\n//|     (moving average of squared gradients).                       |\n//|  epsilon: Small value for numerical stability.                   |\n//+------------------------------------------------------------------+\nOptimizerNadam::OptimizerNadam(double learning_rate=0.010000, double beta1=0.9, double beta2=0.999, double gamma=0.9, double epsilon=1e-8)\n:OptimizerAdam(learning_rate, beta1, beta2, epsilon),\n m_gamma(gamma)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nOptimizerNadam::~OptimizerNadam(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid OptimizerNadam::update(matrix &parameters, matrix &gradients)\n{\n    // Initialize moment and cache matrices if not already initialized\n    if (moment.Rows() != parameters.Rows() || moment.Cols() != parameters.Cols())\n    {\n        moment.Resize(parameters.Rows(), parameters.Cols());\n        moment.Fill(0.0);\n    }\n\n    if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())\n    {\n        cache.Resize(parameters.Rows(), parameters.Cols());\n        cache.Fill(0.0);\n    }\n\n    this.time_step++;\n\n    // Update moment and cache similar to Adam\n    moment = m_beta1 * moment + (1 - m_beta1) * gradients;\n    cache = m_beta2 * cache + (1 - m_beta2) * MathPow(gradients, 2);\n\n    // Bias correction\n    matrix moment_hat = moment / (1 - MathPow(m_beta1, time_step));\n    matrix cache_hat = cache / (1 - MathPow(m_beta2, time_step));\n\n    // Nesterov accelerated gradient\n    matrix nesterov_moment = m_gamma * moment_hat + (1 - m_gamma) * gradients;\n\n    // Update parameters\n    parameters -= m_learning_rate * nesterov_moment / sqrt(cache_hat + m_epsilon);\n}\n\n \n"
  },
  {
    "path": "README.md",
    "content": "<p align=\"center\">\n  <img width=\"25%\" align=\"center\" src=\"https://github.com/MegaJoctan/MALE5/assets/65341461/5a903238-921d-4f09-8e27-1847d4052af3\" alt=\"logo\">\n</p>\n<h1 align=\"center\">\n  M A L E 5\n</h1>\n<p align=\"center\">\n A python-like Machine Learning Library for MQL5\n</p>\n\n<p align=\"center\">\n  <a href=\"https://github.com/MegaJoctan/MALE5/releases\" target=\"_blank\">\n    <img src=\"https://img.shields.io/github/v/release/MegaJoctan/MALE5?color=%2334D058&label=Version\" alt=\"Version\">\n  </a>\n\n  <a href=\"https://github.com/MegaJoctan/MALE5/stargazers\">\n    <img src=\"https://img.shields.io/github/stars/MegaJoctan/MALE5?color=brightgreen&label=Stars\" alt=\"Stars\"/>\n  </a>\n\n  <a href=\"https://github.com/MegaJoctan/MALE5/blob/main/LICENSE\">\n    <img src=\"https://img.shields.io/github/license/MegaJoctan/MALE5?color=blue\" alt=\"License\"/>\n  </a>\n\n  <a>\n    <img src=\"https://img.shields.io/badge/Platform-Win32%20|%20Linux%20|%20macOS-blue?color=blue\" alt=\"Platform Win32 | Linux | macOS\"/>\n  </a>\n\n</p>\n\n<p align=\"center\">\n  <a href=\"https://discord.gg/2qgcadfgrx\" style=\"text-decoration:none\">\n    <img src=\"https://img.shields.io/badge/Discord-%237289DA?style=flat&logo=discord\"/>\n  </a>\n  <a href=\"https://t.me/fxalgebra_discussion\" style=\"text-decoration:none\">\n    <img src=\"https://img.shields.io/badge/Telegram-%232CA5E0?style=flat&logo=telegram\"/>\n  </a>\n</p>\n\n<p align=\"center\">\nEnglish | <a href=\"README_russian.md\">Russian</a> \n</p>\n\n## About the Project\n\nMALE5 is a machine-learning repository for creating trading systems in the c++ like, MQL5 programming language.\nIt was developed to help build machine learning-based trading robots, effortlessly in the [MetaTrader5](https://www.metatrader5.com/en/automated-trading/metaeditor) platform\n\n**My goal is to make the library**\n\n-   **Python-Like, Simple to use** \n-   **Flexible:** To be usable within a Trading Robot(EA), Custom Indicator, and Scripts.\n-   **Resource-efficient:** To make it not consume a lot of resources and memory.\n\n**Disclaimer**\n*This project is not for MQL5 programming language beginners. Those with prior experience of machine learning using python might find this project easy to understand*\n\n## Installing \n\nDownload the zip file from the releases section extract the library. Go under MQL5 folder in your MetaEditor, from there paste the MALE5 directory you extracted under the Include folder.\n\n![bandicam 2024-07-19 16-20-07-392](https://github.com/user-attachments/assets/e2829b7e-8fc5-4829-98fb-9277da2d86ba)\n\n\n## Getting Started with the Library\n\nIn this project, machine learning models can be placed into three categories. *See the tables below*.\n\n## 01: Predictive Models\n\nUnlike others which are mostly used for analysis and data processing. These AI models when given an inputs (predictors) in matrices or vectors they provide predictions. Currently Available models include.\n\n| **Model Type**               | **Models**                       |\n|------------------------------|----------------------------------|\n| **Linear Models**            | - Linear regression              |\n|                              | - Logistic regression            |\n|                              | - Ridge regression               |\n|                              | - Polynomial regression          |\n| **Decision Trees**           | - Decision tree                  |\n| **Ensemble Models**          | - AdaBoost tree                  |\n|                              | - Random forest                  |\n| **Support Vector Machine**   | - SVM                            |\n| **Neural Networks**          | - Regressor Neural Networks      |\n|                              | - Pattern Recognition Neural Networks |\n|                              | - Kohonen Maps                   |\n| **Naïve Bayes**              | - Naïve Bayes models             |\n\n\n\n### Traininng the predictive models\n\nAll the predictive models functions for training and deploying them for predictions follow similar patterns.\n\n[![Watch the video](https://github.com/user-attachments/assets/7d68fa9f-0c75-4d37-9c0f-b9926be9c430)](https://www.youtube.com/watch?v=wKk85PZ2ra8&t=1s)\n\n\n#### For Regression Problems\n\nA regression problem is a type of predictive modeling problem where the goal is to predict a continuous output variable based on one or more input features. In other words, given input data, the model aims to determine a continuous value. For example, predicting the next closing price of a financial instrument.\n\n**How Regression Models Work**\n\nFirstly, we import the necessary libraries we need. In this example, we will be using the Decision Tree Regressor.\n\n```MQL5\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\preprocessing.mqh>\n#include <MALE5\\MatrixExtend.mqh> //helper functions for data manipulations\n#include <MALE5\\metrics.mqh> //for measuring the performance\n\nStandardizationScaler scaler; //standardization scaler from preprocessing.mqh\nCDecisionTreeRegressor *decision_tree; //a decision tree regressor model\n```\n\n1. **Data Collection**\n\nGather a dataset with input features (also called predictors or attributes) and the corresponding target variable (the output).\n\n```MQL5\nvector open, high, low, close;     \nint data_size = 1000;\n\n//--- Getting the open, high, low, and close values for the past 1000 bars, starting from the recent closed bar of 1\nopen.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);\nhigh.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);\nlow.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);\nclose.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);\n\nmatrix X(data_size, 3); //creating the X matrix \n\n//--- Assigning the open, high, and low price values to the X matrix \nX.Col(open, 0);\nX.Col(high, 1);\nX.Col(low, 2);\n\nvector y = close; // The target variable is the close price\n```\n\n2. **Preprocessing**\n\nThis involves cleaning and preprocessing the data, which might involve handling missing values, normalizing the data, and encoding categorical variables if possible in the data. \n\n```MQL5\n//--- We split the data into training and testing samples for training and evaluation\nmatrix X_train, X_test;\nvector y_train, y_test;\n\ndouble train_size = 0.7; //70% of the data should be used for training, the rest for testing\nint random_state = 42; //we put a random state to shuffle the data\n\nMatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the X and y data into training and testing samples         \n\n//--- Normalizing the independent variables\nX_train = scaler.fit_transform(X_train); // fit the scaler on the training data and transform the data\nX_test = scaler.transform(X_test); // transform the test data\n\n// Print the processed data for verification\n//Print(\"X_train\\n\",X_train,\"\\nX_test\\n\",X_test,\"\\ny_train\\n\",y_train,\"\\ny_test\\n\",y_test); \n```\n\n3. **Model Selection**\n\nChoose a regression algorithm. Common algorithms for this type of problem include linear regression, decision trees, support vector regression (SVR), k-nearest neighbors (K-NN), and neural networks.\n\n```MQL5\n//--- Model selection\ndecision_tree = new CDecisionTreeRegressor(2, 5); //a decision tree regressor from DecisionTree class\n```\n\n4. **Training**\n\nUse the training data to teach the model how to map input features to the correct continuous output values. During training, the model learns patterns in the data.\n\n```MQL5\n//--- Training the model\ndecision_tree.fit(X_train, y_train); // The training function\n```\n\n5. **Evaluation**\n\nAssess the model's performance on a separate test dataset to ensure it can generalize well to new, unseen data. Metrics like R-squared (R²) score, mean absolute error (MAE), and root mean squared error (RMSE) are often used to evaluate the model.\n\n```MQL5\n//--- Measuring predictive accuracy \nvector train_predictions = decision_tree.predict(X_train);\nprintf(\"Decision Tree training R² score = %.3f \", Metrics::r_squared(y_train, train_predictions));\n\n//--- Evaluating the model on out-of-sample predictions\nvector test_predictions = decision_tree.predict(X_test);\nprintf(\"Decision Tree out-of-sample R² score = %.3f \", Metrics::r_squared(y_test, test_predictions));\n```\n\n6. **Prediction**\n\nOnce trained and evaluated, the model can be used to predict the continuous values of new, unseen data points.\n\n```MQL5\n\nvoid OnTick()\n{\n   //--- Making predictions live from the market \n   CopyRates(Symbol(), PERIOD_D1, 1, 1, rates); // Get the very recent information from the market\n   \n   vector x = {rates[0].open, rates[0].high, rates[0].low}; // Assigning data from the recent candle in a similar way to the training data\n   \n   double predicted_close_price = decision_tree.predict(x);\n   \n   Comment(\"Next closing price predicted is = \", predicted_close_price);  \n}\n\n```\n\n**Types of Regression**\n\n- **Simple Linear Regression:** Predicting a continuous target variable based on a single input feature.\n- **Multiple Linear Regression:** Predicting a continuous target variable based on multiple input features.\n- **Polynomial Regression:** Predicting a continuous target variable using polynomial terms of the input features.\n- **Non-Linear Regression:** Predicting a continuous target variable using non-linear relationships between input features and the target variable.\n\n\n\n#### For Classification Problems\n\nA classification problem is a type of predictive modeling problem where the goal is to predict the category or class that a given data point belongs to. In other words, given input data, the model aims to determine which of the predefined classes the input belongs to. Forexample when trying to predict whether the next candle will be either bullish or bearish.\n\n**How Classification Models Work**\n\nFirstly, we import the necessary libraries we need. In this example we will be using the Decision Tree Classifier.\n\n```MQL5\n\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\preprocessing.mqh>\n#include <MALE5\\MatrixExtend.mqh> //helper functions for data manipulations\n#include <MALE5\\metrics.mqh> //fo measuring the performance\n\nStandardizationScaler scaler; //standardization scaler from preprocessing.mqh\nCDecisionTreeClassifier *decision_tree; //a decision tree classifier model\n\n```\n\n1. **Data Collection** \n\nGather a dataset with input features (also called predictors or attributes) and the corresponding class labels (the output).\n> \n\n```MQL5\n\n     vector open, high, low, close;     \n     int data_size = 1000;\n     \n//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1\n     \n     open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);\n     high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);\n     low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);\n     close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);\n     \n     decision_tree = new CDecisionTreeClassifier(2, 5);\n     \n     matrix X(data_size, 3); //creating the x matrix \n   \n//--- Assigning the open, high, and low price values to the x matrix \n\n     X.Col(open, 0);\n     X.Col(high, 1);\n     X.Col(low, 2);\n     \n```\n\n2. **Preprocessing**\n\nThis involves clearning and preprocess the data, which might involve analytical techniques such as handling missing values, **normalizing the data**, and encoding categorical variables.\n\n```MQL5\n\n//--- We split the data into training and testing samples for training and evaluation\n \n     matrix X_train, X_test;\n     vector y_train, y_test;\n     \n     double train_size = 0.7; //70% of the data should be used for training the rest for testing\n     int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable\n      \n     MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples         \n     \n\n//--- Normalizing the independent variables\n   \n     X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether\n     X_test = scaler.transform(X_test); // we transform the new data this way\n     \n     //Print(\"X_train\\n\",X_train,\"\\nX_test\\n\",X_test,\"\\ny_train\\n\",y_train,\"\\ny_test\\n\",y_test); \n\n```\n\n3. **Model Selection** \n\nChoose a classification algorithm. Common algorithms for this type of problem include; logistic regression, decision trees, support vector machines (SVM), k-nearest neighbors (K-NN), and neural networks.\n\n```MQL5\n\n//--- Model selection\n   \n     decision_tree = new CDecisionTreeClassifier(2, 5); //a decision tree classifier from DecisionTree class\n\n```\n\n4. **Training** \n\nUse the training data to teach the model how to map input features to the correct class labels. During training, the model learns patterns in the data.\n\n```MQL5\n\n//--- Training the  model\n     \n     decision_tree.fit(X_train, y_train); //The training function \n     \n```\n\n5. **Evaluation**\n\nAssess the model's performance on a separate test dataset to ensure it can generalize well to new, unseen data. Metrics like accuracy score, precision, recall, and F1 score are often used to evaluate the model.\n\n```MQL5\n\n//--- Measuring predictive accuracy \n   \n     vector train_predictions = decision_tree.predict_bin(X_train);\n     \n     printf(\"Decision decision_tree training accuracy = %.3f \",Metrics::accuracy_score(y_train, train_predictions));\n\n//--- Evaluating the model on out-of-sample predictions\n     \n     vector test_predictions = decision_tree.predict_bin(X_test);\n     \n     printf(\"Decision decision_tree out-of-sample accuracy = %.3f \",Metrics::accuracy_score(y_test, test_predictions)); \n\n\n```\n\n6. **Prediction** \n\nOnce trained and evaluated, the model can be used to predict the class labels of new, unseen data points.\n\nUnlike the `predict` function which is used to obtain predictions in regressors. The predictive functions for the classifiers have different slightly for different name starting with the word predict.\n\n- **predict_bin** - This function can be used to predict the classes in as integer values for classification models. For example: The next candle will be ***bullish*** or ***bearish***\n- **predict_proba** - This function predicts the probabilities of a certain classes as double values from 0 for a 0% probability chance to 1 for a 100% probability chance. For example: [0.64, 0.36] probability the next candle will be bullish is 64%, while the probability the next candle will be bearish is 36%\n\n```\nvoid OnTick()\n  {\n     \n//--- Making predictions live from the market \n   \n   CopyRates(Symbol(), PERIOD_D1, 1, 1, rates); //Get the very recent information from the market\n   \n   vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data\n   \n   double predicted_close_price = decision_tree.predict_bin(x);\n   \n   Comment(\"Next closing price predicted is = \",predicted_close_price);  \n  }\n\n```\n\n**Types of Classification**\n\n- **Binary Classification:** There are only two classes. For example, classifying emails as spam or not spam.\n- **Multiclass Classification:** There are more than two classes. For example, classifying types of animals in an image (e.g., cats, dogs, birds).\n- **Multilabel Classification:** Each instance can be assigned multiple classes. For example, tagging a photo with multiple labels like \"beach\", \"sunset\", and \"vacation\".\n\n\n\n\n## 02: Clustering Algorithms\n\nThese algorithms are for the special purppose of classifying and grouping data with similar patterns and contents together, they excel in data mining situations\n\n| **Model Type**               | **Models**                       |\n|------------------------------|----------------------------------|\n| **Nearest Neighbors**        | - KNN nearest neighbors          |\n| **K-Means Clustering**       | - k-Means clustering algorithm   |\n| **Neural Networks**          | - Kohonen Maps                   |\n\n## 03: Dimensionality Reduction Algorithms\n\nThese algorithms are widely used in situations where the dataset is huge and needs adjustmets. they excel at reducing the size of datasets without losing much information. Consider them as the algorithms to zip and shread information in the proper way.\n\n| **Model Type**                         | **Models**                          |\n|----------------------------------------|-------------------------------------|\n| **Dimensionality Reduction Algorithms**| - Linear Discriminant Analysis (LDA)|\n|                                        | - Non Negative Matrix Factorization (NMF) |\n|                                        | - Principal Component Analysis (PCA) |\n|                                        | - Truncated SVD                     |\n\n\n\n## Basic Library functionality & helpers\n\n* [MatrixExtend (MatrixExtend.mqh)](https://github.com/MegaJoctan/MALE5/wiki#matrixextendmatrixextendmqh)\n* [Cross Validation Library (cross_validation.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Cross-Validation-Library)\n* [Linear Algebra Library (linalg.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Linear-Algebra-Library)\n* [Kernels library (kernels.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Kernels-Library)\n* [Metrics library (metrics.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Metrics-library)\n* [Pre-processing library (preprocessing.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Pre-processing-library)\n* [Tensor library (Tensor.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Tensor-Library)\n\n\n\n## Opening an issue\nYou can also post bug reports and feature requests (only) in [GitHub issues](https://github.com/MegaJoctan/MALE5/issues).\n\n## Support the Project\nIf you find this project helpful, Support us by taking one or more of the actions\n\n[BuyMeCoffee](https://www.buymeacoffee.com/omegajoctan)\n\n[OurProducts](https://www.mql5.com/en/users/omegajoctan/seller)\n\nRegister to our recommended broker:\n\n[ICMarkets](https://icmarkets.com/?camp=74639)\n\n## Let's work together\n[HIRE ME on MQL5.com by clicking on this link](https://www.mql5.com/en/job/new?prefered=omegajoctan)\n\n\n"
  },
  {
    "path": "Sklearn/Cluster/DBSCAN.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                       DBSCAN.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n#include <MALE5\\MatrixExtend.mqh>\n#include <MALE5\\linalg.mqh>\n\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\nclass CDBSCAN\n  {\nprotected:\n   double m_epsilon;\n   uint m_minsamples;\n   vector labels;\n   \n   vector get_neighbors(matrix &x, ulong point_index);\n   bool expand_cluster(matrix &x, ulong point_index, ulong cluster_id, const vector &neighbors, const vector &cluster_labels);\n   \npublic:\n                     CDBSCAN(double epsilon, uint min_samples);\n                    ~CDBSCAN(void);\n                    \n                    vector fit_predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDBSCAN::CDBSCAN(double epsilon, uint min_samples):\nm_epsilon(epsilon),\nm_minsamples(min_samples)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDBSCAN::~CDBSCAN(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CDBSCAN::get_neighbors(matrix &x, ulong point_index)\n {\n   vector neighbors = {};\n   \n   for (ulong i=0, count=0; i<x.Rows(); i++)\n    {\n      if (LinAlg::norm(x.Row(point_index), x.Row(i)) < this.m_epsilon && i != point_index)\n       { \n         count++;\n         neighbors.Resize(count);\n         neighbors[count-1] = (int)i; //Append the neighbor\n       }\n    }\n   \n   return neighbors;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CDBSCAN::expand_cluster(matrix &x, ulong point_index, ulong cluster_id, const vector &neighbors, const vector &cluster_labels)\n {\n \n   this.labels[point_index] = (int)cluster_id;\n   vector points_to_explore = {(int)point_index};\n   \n   Print(\"points to explore in a cluster: \",points_to_explore);\n   \n   while (points_to_explore.Size()>0)\n     {\n       vector current_point = {points_to_explore[0]};\n       points_to_explore.Resize(0);\n       \n       Print(\"current point: \", current_point);\n       \n       vector current_neighbors = this.get_neighbors(x, (int)current_point[0]);\n       \n       if (current_neighbors.Size() >= this.m_minsamples)\n         for (int neighbor_index=0; neighbor_index<(int)current_neighbors.Size(); neighbor_index++)\n           {\n             if (this.labels[neighbor_index] == 0 || this.labels[neighbor_index] == -1)\n               {\n                  if (this.labels[neighbor_index] == 0)\n                    {\n                      points_to_explore.Resize(1);\n                      points_to_explore[0] = neighbor_index;\n                      \n                      Print(\"points to explore: \",points_to_explore);\n                    }\n                 this.labels[neighbor_index] = (int)cluster_id;\n               }  \n           }\n     }\n   \n   return true; \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CDBSCAN::fit_predict(matrix &x)\n {\n   this.labels.Resize(x.Rows()); labels.Fill(0);\n   ulong cluster_id = 0; \n   \n   for (ulong i=0; i<x.Rows(); i++) \n     {\n       if (this.labels[i] != 0)\n         continue;\n        \n        vector neighbors = get_neighbors(x, i);\n        \n        \n        if (neighbors.Size() < this.m_minsamples)\n          this.labels[i] = -1; //Mark as noise\n        else\n          {\n           cluster_id++;\n           Print(\"Expand cluster: \",i);\n           expand_cluster(x, i, cluster_id, neighbors, this.labels);\n          }  \n     }\n   return this.labels;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n"
  },
  {
    "path": "Sklearn/Cluster/Hierachical Clustering.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                       Hierachical Clustering.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n#include \"Base.mqh\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n\nenum linkage_enum \n {\n   LINKAGE_SINGLE,\n   LINKAGE_COMPLETE,\n   LINKAGE_AVG\n };\n \nclass CAgglomerativeClustering\n  {\nprotected:\n\n   linkage_enum linkage;\n   vector       labels;\n   matrix       clusters_keys;\n   \n   matrix       calc_distance_matrix(matrix &x, vector &cluster_members);   \n   \npublic:\n                     CAgglomerativeClustering(linkage_enum linkage_type=LINKAGE_SINGLE);\n                    ~CAgglomerativeClustering(void);\n                    \n                    void fit(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCAgglomerativeClustering::CAgglomerativeClustering(linkage_enum linkage_type=LINKAGE_SINGLE)\n :linkage(linkage_type)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCAgglomerativeClustering::~CAgglomerativeClustering(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CAgglomerativeClustering::calc_distance_matrix(matrix &x, vector &cluster_members)\n {\n   clusters_keys.Init(1, x.Cols()); //initializes the clusters_keys such that each data point is initially in its own cluster\n   for (ulong i=0; i<x.Cols(); i++)    clusters_keys.Col(clusters_keys.Col(i).Fill(i), i); //Filll the initial clusters_keys matrix with their columns incremental values\n   \n   matrix distance(x.Rows(), x.Rows());\n   distance.Fill(0.0);\n   \n   vector v1, v2;\n   \n   vector ith_element, jth_element;\n   for (ulong i=0; i<distance.Cols(); i++)\n    {\n     ith_element = cluster_members[(int)clusters_keys[i]];\n     for (ulong j=0; j<distance.Cols(); j++)\n        {\n          jth_element = cluster_members[(int)clusters_keys[j]];\n          \n          v1 = x.Col(i); v2 = x.Col(j);\n          distance[i][j] = Base::norm(v1, v2);\n        }\n    }\n   \n   return distance;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CAgglomerativeClustering::fit(matrix &x)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Cluster/KMeans.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                       KMeans.mqh |\n//|                                    Copyright 2022, Omega Joctan. |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Omega Joctan.\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n\n#include <Graphics\\Graphic.mqh>\nCGraphic graph;\n#include <MALE5\\MatrixExtend.mqh>\n\n//+------------------------------------------------------------------+\nenum errors\n  {\n   KM_ERR001, //clusters not matching in size Error\n   KM_ERR002\n  };\n\n//+------------------------------------------------------------------+\n \nclass CKMeans\n  {\n  \nprivate:\n   ulong             n; //number of samples\n   uint              m_clusters;\n   ulong             m_cols;\n   matrix            InitialCentroids;\n   vector<double>    cluster_assign;\n\nprotected:\n   matrix            Matrix;\n   bool              ErrMsg(errors err); \n   bool              ScatterCurvePlots(\n                                       string obj_name,\n                                       double &x[],\n                                       double &y[],\n                                       double &curveArr[],\n                                       string legend,\n                                       string x_axis_label = \"x-axis\",\n                                       string y_axis_label = \"y-axis\",\n                                       color  clr = clrDodgerBlue,\n                                       bool   points_fill = true\n                                    );\n\npublic:\n                     CKMeans(const matrix &_Matrix, int clusters=3);\n                    ~CKMeans(void);\n\n   void              KMeansClustering(matrix &clustered_matrix, matrix &centroids, int iterations = 1, bool rand_cluster =false);\n   void              ElbowMethod(const int initial_k=1, int total_k=10, bool showPlot = true);\n   void              FilterZero(vector &vec);\n   void              matrixtoArray(matrix &mat, double &Array[]);\n  };\n//+------------------------------------------------------------------+\nCKMeans::CKMeans(const matrix &_Matrix, int clusters=3)\n  {\n   m_clusters = clusters;\n   Matrix.Copy(_Matrix);\n\n   m_cols = Matrix.Cols();\n   n = Matrix.Rows(); //number of elements | Matrix Rows\n  }\n//+------------------------------------------------------------------+\nCKMeans::~CKMeans(void)\n  {\n   ZeroMemory(m_clusters);\n   ZeroMemory(InitialCentroids);\n   ZeroMemory(cluster_assign);\n  }\n\n//+------------------------------------------------------------------+\n \nvoid CKMeans::KMeansClustering(matrix &clustered_matrix, matrix &centroids, int iterations = 1, bool rand_cluster =false)\n  {\n   InitialCentroids.Resize(m_clusters, m_cols);\n   cluster_assign.Resize(n);\n   \n   clustered_matrix.Resize(m_clusters, m_cols*n);\n   clustered_matrix.Fill(NULL);\n\n   vector cluster_comb_v = {};\n   matrix cluster_comb_m = {};\n\n   vector rand_v = {};\n   ulong rand_ = 0;\n\n   for(ulong i=0; i<m_clusters; i++)\n     {\n      rand_ = rand_cluster ? (ulong)MathFloor(i*(n/m_clusters)) : i;\n      rand_v = Matrix.Row(rand_);\n\n      InitialCentroids.Row(rand_v, i);\n     }\n\n//---\n\n   vector v_row;\n\n\n   matrix rect_distance = {};  //matrix to store rectilinear distances\n   rect_distance.Reshape(n, m_clusters);\n\n\n   vector v_matrix = {}, v_centroid = {};\n   double output = 0;\n\n//---\n\n   for(int iter=0; iter<iterations; iter++)\n     {\n\n      for(ulong i=0; i<rect_distance.Rows(); i++)\n         for(ulong j=0; j<rect_distance.Cols(); j++)\n           {\n            v_matrix = Matrix.Row(i);\n            v_centroid = InitialCentroids.Row(j);\n\n            ZeroMemory(output);\n\n            for(ulong k=0; k<v_matrix.Size(); k++)\n               output += MathAbs(v_matrix[k] - v_centroid[k]); //Rectilinear distance\n\n            rect_distance[i][j] = output;\n           }\n\n      //---  Assigning the Clusters\n\n      matrix cluster_cent = {}; //cluster centroids\n      ulong cluster = 0;\n\n      for(ulong i=0; i<rect_distance.Rows(); i++)\n        {\n         v_row = rect_distance.Row(i);\n         cluster = v_row.ArgMin();\n\n         cluster_assign[i] = (double)cluster;\n        }\n\n\n      vector temp_cluster_assign = cluster_assign;\n\n      //--- Combining the clusters\n\n      for(ulong i=0, index =0; i<cluster_assign.Size(); i++)\n        {\n         ZeroMemory(cluster_cent);\n         bool classified = false;\n\n         for(ulong j=0, count = 0; j<temp_cluster_assign.Size(); j++)\n           {\n\n            if(cluster_assign[i] == temp_cluster_assign[j])\n              {\n               classified = true;\n\n               count++;\n\n               cluster_comb_m.Resize(count, m_cols);\n\n               cluster_comb_m.Row(Matrix.Row(j), count-1);\n\n               cluster_cent.Resize(count, m_cols);\n\n               // New centroids\n               cluster_cent.Row(Matrix.Row(j), count-1);\n\n               temp_cluster_assign[j] = -100; //modify the cluster item so it can no longer be found\n              }\n            else\n               continue;\n           }\n\n         //---\n\n         if(classified == true)\n           {\n            // solving for new cluster and updtating the old ones\n            \n            \n             cluster_comb_v = MatrixExtend::MatrixToVector(cluster_comb_m);\n            \n\n            if(iter == iterations-1)\n               clustered_matrix.Row(cluster_comb_v, index); \n\n            index++;\n            //---\n\n            vector x_y_z = {0, 0};\n            ZeroMemory(rand_v);\n\n            for(ulong k=0; k<cluster_cent.Cols(); k++)\n              {\n               x_y_z.Resize(cluster_cent.Cols());\n               rand_v = cluster_cent.Col(k);\n\n               x_y_z[k] = rand_v.Mean();\n              }\n\n            InitialCentroids.Row(x_y_z, i);\n\n           }\n\n         if(index >= m_clusters)\n            break;\n        }\n\n     } //end of iterations\n\n   ZeroMemory(centroids);\n   centroids.Copy(InitialCentroids);\n  }\n\n//+------------------------------------------------------------------+\n\nbool CKMeans::ErrMsg(errors err)\n  {\n   switch(err)\n     {\n\n      case  KM_ERR001:\n         printf(\"%s Clusters not matching in Size \", EnumToString(KM_ERR001));\n         break;\n      default:\n         break;\n     }\n   return(true);\n  }\n\n//+------------------------------------------------------------------+\n \nvoid CKMeans::ElbowMethod(const int initial_k=1, int total_k=10, bool showPlot = true)\n  {\n   matrix clustered_mat, _centroids = {};\n\n   if(total_k > (int)n)\n      total_k = (int)n; //k should always be less than n\n\n   vector centroid_v= {}, x_y_z= {};\n   vector short_v = {}; //vector for each point\n   vector minus_v = {}; //vector to store the minus operation output\n\n   double wcss = 0;\n   double WCSS[];\n   ArrayResize(WCSS, total_k);\n   double kArray[];\n   ArrayResize(kArray, total_k);\n\n   for(int k=initial_k, count_k=0; k<ArraySize(WCSS)+initial_k; k++, count_k++)\n     {\n\n      wcss = 0;\n\n      m_clusters = k;\n\n      KMeansClustering(clustered_mat, _centroids);\n\n      for(ulong i=0; i<_centroids.Rows(); i++)\n        {\n         centroid_v = _centroids.Row(i);\n\n         x_y_z = clustered_mat.Row(i);\n         FilterZero(x_y_z);\n\n\n         for(ulong j=0; j<x_y_z.Size()/m_cols; j++)\n           {\n\n            MatrixExtend::Copy(x_y_z, short_v, uint(j*m_cols), (uint)m_cols);\n\n            //---                WCSS ( within cluster sum of squared residuals )\n\n            minus_v = (short_v - centroid_v);\n\n            minus_v = MathPow(minus_v, 2);\n\n            wcss += minus_v.Sum();\n\n           }\n\n        }\n\n      WCSS[count_k] = wcss;\n      kArray[count_k] = k;\n     }\n\n   Print(\"WCSS\");\n   ArrayPrint(WCSS);\n   Print(\"kArray\");\n   ArrayPrint(kArray);\n\n//--- Plotting the Elbow on the graph\n\n   if(showPlot)\n     {\n      ObjectDelete(0, \"elbow\");\n      ScatterCurvePlots(\"elbow\", kArray, WCSS, WCSS, \"Elbow line\", \"k\", \"WCSS\");\n     }\n  }\n\n//+------------------------------------------------------------------+\n \nvoid CKMeans::FilterZero(vector &vec)\n  {\n   vector new_vec = {};\n\n   for(ulong i=0, count =0; i<vec.Size(); i++)\n     {\n      if(vec[i] != NULL)\n        {\n         count++;\n         new_vec.Resize(count);\n         new_vec[count-1] = vec[i];\n        }\n      else\n         continue;\n     }\n\n   vec.Copy(new_vec);\n  }\n\n//+------------------------------------------------------------------+\n//+------------------------------------------------------------------+\n \nbool CKMeans::ScatterCurvePlots(\n   string obj_name,\n   double &x[],\n   double &y[],\n   double &curveArr[],\n   string legend,\n   string x_axis_label = \"x-axis\",\n   string y_axis_label = \"y-axis\",\n   color  clr = clrDodgerBlue,\n   bool   points_fill = true\n)\n  {\n\n   if(!graph.Create(0, obj_name, 0, 30, 70, 800, 640))\n     {\n      printf(\"Failed to Create graphical object on the Main chart Err = %d\", GetLastError());\n      return(false);\n     }\n\n   ChartSetInteger(0, CHART_SHOW, true);\n\n//---\n\n//graph.CurveAdd(x,y,clrBlack,CURVE_POINTS,y_axis_label);\n   graph.CurveAdd(x, curveArr, clr, CURVE_POINTS_AND_LINES, legend);\n\n   graph.XAxis().Name(x_axis_label);\n   graph.XAxis().NameSize(13);\n   graph.YAxis().Name(y_axis_label);\n   graph.YAxis().NameSize(13);\n   graph.FontSet(\"Lucida Console\", 13);\n   graph.CurvePlotAll();\n   graph.Update();\n\n   return(true);\n  }\n\n//+------------------------------------------------------------------+\n \nvoid CKMeans::matrixtoArray(matrix &mat, double &Array[])\n  {\n   ArrayFree(Array);\n   ArrayResize(Array, int(mat.Rows()*mat.Cols()));\n\n   int index = 0;\n   for(ulong i=0; i<mat.Rows(); i++)\n      for(ulong j=0; j<mat.Cols(); j++, index++)\n        {\n         Array[index] = mat[i][j];\n        }\n  }\n\n//+------------------------------------------------------------------+ \n"
  },
  {
    "path": "Sklearn/Cluster/base.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                         BaseClustering.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n#include <MALE5\\MatrixExtend.mqh>\n#include <MALE5\\linalg.mqh>\n#include <MALE5\\MqPlotLib\\plots.mqh>\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\nclass BaseClustering\n  {\npublic:\n                     BaseClustering(void);\n                    ~BaseClustering(void);\n                    \n                    static matrix Get(matrix &X, vector &index, int axis=0);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix BaseClustering::Get(matrix &X, vector &index, int axis=0)\n {\n   matrix ret_matrix = {};\n   \n   ulong row_col=0;\n   bool isRows = true;\n    switch(axis)\n      {\n       case  0:\n         row_col = X.Rows();\n         isRows = true;\n         break;\n       case 1:\n         row_col = X.Cols();\n         isRows = false;\n         break;\n       default:\n         printf(\"%s Invalid axis %d \",__FUNCTION__,axis);\n         return ret_matrix;\n         break;\n      }\n//---\n   \n   ret_matrix.Resize(isRows?index.Size():X.Rows(), isRows?X.Cols(): index.Size());\n   \n   for (ulong i=0, count=0; i<row_col; i++)\n     for (ulong j=0; j<index.Size(); j++)\n        {\n          if (isRows)\n           {\n             if (i==index[j])\n              {\n                if (isRows)\n                  ret_matrix.Row(X.Row(i), count);\n                else\n                  ret_matrix.Col(X.Col(i), count);\n                  \n                count++;\n              }\n           }\n        }\n     \n  return ret_matrix; \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Sklearn/Decomposition/LDA.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                          LDA.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n\n#include \"base.mqh\";\n#include <MALE5\\Numpy\\Numpy.mqh>\n#include <MALE5\\MqPlotLib\\plots.mqh>\n\nenum lda_criterion //selecting best components criteria selection\n  {\n    CRITERION_VARIANCE,\n    CRITERION_KAISER,\n    CRITERION_SCREE_PLOT\n  };\n\nclass CLDA\n  {  \nCPlots   plt;\nCNumpy np;\n\nprotected:\n   uint m_components;\n   lda_criterion m_criterion;\n   \n   matrix projection_matrix;\n   ulong num_features;\n   double m_regparam;\n   vector mean;\n   \n   uint calculate_variance(vector &eigen_values, double threshold=0.95);\n   uint calculate_kaiser(vector &eigen_values);\n   \n   uint CLDA::extract_components(vector &eigen_values, double threshold=0.95);\n   \npublic:\n                     CLDA(uint k=NULL, lda_criterion CRITERION_=CRITERION_SCREE_PLOT, double reg_param =1e-6);\n                    ~CLDA(void);\n                    \n                     matrix fit_transform(const matrix &x, const vector &y);\n                     matrix transform(const matrix &x);\n                     vector transform(const vector &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCLDA::CLDA(uint k=NULL, lda_criterion CRITERION_=CRITERION_SCREE_PLOT, double reg_param=1e-6)\n:m_components(k),\n m_criterion(CRITERION_),\n m_regparam(reg_param)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCLDA::~CLDA(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix  CLDA::fit_transform(const matrix &x, const vector &y)\n {\n   vector classes = np.unique(y).unique;\n   ulong num_classes = classes.Size();\n   num_features = x.Cols();\n   \n   this.mean = x.Mean(0);\n   \n   matrix x_centered = BaseDimRed::subtract(x, this.mean);\n   \n   matrix class_means(classes.Size(), x.Cols());\n   class_means.Fill(0.0);\n   \n   for (ulong i=0; i<num_classes; i++)\n    {\n     matrix class_samples = {};\n      for (ulong j=0, count=0; j<x.Rows(); j++)\n         {\n           if (y[j] == classes[i])\n            {  \n               count++;\n               class_samples.Resize(count, num_features);\n               class_samples.Row(x.Row(j), count-1);\n            }\n         }\n         \n      class_means.Row(class_samples.Mean(0), i);\n    }\n    \n    \n  matrix SW, SB; //within and between scatter matrices \n  SW.Init(num_features, num_features);\n  SB.Init(num_features, num_features);\n  \n  for (ulong i=0; i<num_classes; i++)\n   {\n     matrix class_samples = {};\n      for (ulong j=0, count=0; j<x.Rows(); j++)\n         {\n           if (y[j] == classes[i]) //Collect a matrix for samples belonging to a particular class\n            {\n               count++;\n               class_samples.Resize(count, num_features);\n               class_samples.Row(x.Row(j), count-1);\n            }\n         }\n\n         \n     matrix diff = BaseDimRed::subtract(class_samples, class_means.Row(i)); //Each row subtracted to the mean\n     if (diff.Rows()==0 && diff.Cols()==0) //if the subtracted matrix is zero stop the program for possible bugs or errors\n      {\n        DebugBreak();\n        return x_centered;\n      }\n     \n     SW += diff.Transpose().MatMul(diff); //Find within scatter matrix \n     \n     vector mean_diff = class_means.Row(i) - x_centered.Mean(0);\n     SB += class_samples.Rows() * mean_diff.Outer(mean_diff); //compute between scatter matrix \n   }\n  \n//--- Regularization to avoid errors while calculating Eigen values and vectors\n   \n   SW += this.m_regparam * np.eye((uint)num_features, (uint)num_features, 1); \n   SB += this.m_regparam * np.eye((uint)num_features, (uint)num_features, 1);\n\n//---\n\n  matrix eigen_vectors(x.Cols(), x.Cols());  eigen_vectors.Fill(0.0);\n  vector eigen_values(x.Cols()); eigen_values.Fill(0.0);\n  \n  matrix SBSW = SW.Inv().MatMul(SB);\n  \n  BaseDimRed::ReplaceNaN(SBSW);\n  \n  if (!SBSW.Eig(eigen_vectors, eigen_values))\n    {\n      Print(\"%s Failed to calculate eigen values and vectors Err=%d\",__FUNCTION__,GetLastError());\n      DebugBreak();\n      \n      matrix empty = {};\n      return empty;\n    }\n    \n//--- Sort eigenvectors by decreasing eigenvalues\n   \n   vector args = np.argsort(eigen_values);\n   args = np.reverse(args);\n   \n   eigen_values = BaseDimRed::Sort(eigen_values, args);\n   eigen_vectors = BaseDimRed::Sort(eigen_vectors, args);\n   \n//---\n   \n  if (this.m_components == NULL)\n   {\n     this.m_components = extract_components(eigen_values);\n     if (this.m_components==0)\n      {\n        printf(\"%s Failed to auto detect the best components\\n You need to select the value of k yourself by looking at the scree plot\",__FUNCTION__);\n        this.m_components = (uint)x.Cols();\n      }\n   }\n  else //plot the scree plot \n    extract_components(eigen_values);\n    \n  this.projection_matrix = BaseDimRed::Slice(eigen_vectors, this.m_components);\n    \n  return x_centered.MatMul(projection_matrix.Transpose());\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CLDA::transform(const matrix &x)\n {\n   if (this.projection_matrix.Rows() == 0)\n    {\n      printf(\"%s fit_transform method must be called befor transform\",__FUNCTION__);\n      matrix empty = {};\n      return empty; \n    }\n  matrix x_centered = BaseDimRed::subtract(x, this.mean);\n  \n  return x_centered.MatMul(this.projection_matrix.Transpose());  \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CLDA::transform(const vector &x)\n {\n   matrix m = CUtils::VectorToMatrix(x, this.num_features); \n   \n   if (m.Rows()==0)\n    {\n      vector empty={};\n      return empty; //return nothing since there is a failure in converting vector to matrix\n    }\n   \n   m = transform(m);\n   return np.flatten(m);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint CLDA::calculate_variance(vector &eigen_values, double threshold=0.95)\n {\n  uint k=0; \n  \n   vector eigen_pow = MathPow(eigen_values, 2);\n   vector cum_sum = eigen_pow.CumSum();\n   double sum = eigen_pow.Sum();\n   \n   vector cumulative_variance =  cum_sum / sum;\n   \n   if (MQLInfoInteger(MQL_DEBUG))\n     Print(\"Cummulative variance: \",cumulative_variance);\n   \n   vector v(cumulative_variance.Size());  v.Fill(0.0);\n   for (ulong i=0; i<v.Size(); i++)\n     v[i] = (cumulative_variance[i] >= threshold);\n      \n   k = (uint)v.ArgMax() + 1;\n   \n   return k;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint CLDA::calculate_kaiser(vector &eigen_values)\n {\n  vector v(eigen_values.Size()); v.Fill(0.0);\n   for (ulong i=0; i<eigen_values.Size(); i++)\n     v[i] = (eigen_values[i] >= 1);\n   \n   return uint(v.Sum());\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint CLDA::extract_components(vector &eigen_values, double threshold=0.95)\n {\n  uint k = 0;\n  \n   switch(m_criterion)\n     {\n      case  CRITERION_VARIANCE: \n         k = calculate_variance(eigen_values, threshold);\n         \n        break;\n        \n      case  CRITERION_KAISER:\n         k = calculate_kaiser(eigen_values);\n        \n        break;\n        \n      case  CRITERION_SCREE_PLOT:\n       {  \n         vector v_cols(eigen_values.Size());\n         \n         for (ulong i=0; i<v_cols.Size(); i++)\n             v_cols[i] = (int)i+1;\n             \n          vector vars = eigen_values;\n          \n          plt.Plot(\"Scree plot\",v_cols,vars,\"EigenValue\",\"Principal Components\",\"EigenValue\", CURVE_POINTS_AND_LINES);\n\n//---\n      string warn = \"\\n<<<< WARNING >>>>\\nThe Scree plot doesn't return the determined number of k m_components\\nThe cummulative variance Or kaiser will return the number of k m_components instead\\nThe k returned might be different from what you see on the scree plot\";\n             warn += \"\\nTo apply the same number of k m_components to the LDA from the scree plot\\nCall the LDA model again with that value applied from the plot\\n\";\n      \n         Print(warn);\n        \n        //--- Kaiser\n           \n           k = calculate_kaiser(eigen_values);\n            \n            if (k==0) //kaiser wasn't suitable in this particular task\n              k = calculate_variance(eigen_values, threshold);\n        }          \n           \n        break;\n     } \n     \n   return (k);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n"
  },
  {
    "path": "Sklearn/Decomposition/NMF.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                          NMF.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include \"base.mqh\";\n#include <MALE5\\Numpy\\Numpy.mqh>\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass CNMF\n  {\nprotected:\n\n   CNumpy np;\n   uint m_components;\n   uint m_max_iter;\n   int m_randseed;\n   ulong n_features;\n   matrix W; //Basic matrix\n   matrix H; //coefficient matrix\n   double m_tol; //loss tolerance\n   \npublic:\n                     CNMF(uint max_iter=100, double tol=1e-4, int random_state=-1);\n                    ~CNMF(void);\n                    \n                    matrix fit_transform(matrix &X, uint k=2);\n                    matrix transform(matrix &X);\n                    vector transform(vector &X);\n                    uint select_best_components(matrix &X);\n                    \n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCNMF::CNMF(uint max_iter=100, double tol=1e-4,int random_state=-1)\n :m_max_iter(max_iter),\n m_randseed(random_state),\n m_tol(tol)\n {\n   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCNMF::~CNMF(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CNMF::transform(matrix &X)\n {\n  n_features = X.Cols();\n  if (m_components>n_features)\n     {\n       printf(\"%s Number of dimensions K[%d] is supposed to be <= number of features %d\",__FUNCTION__,m_components,n_features);\n       this.m_components = (uint)n_features;\n     }\n     \n  if (this.W.Rows()==0 || this.H.Rows()==0)\n    {\n      Print(__FUNCTION__,\" Model not fitted. Call fit method first.\");\n      matrix mat={};\n      return mat;\n    }\n  \n  return X.MatMul(this.H.Transpose());\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CNMF::transform(vector &X)\n {\n   matrix INPUT_MAT = np.expand_dims(X, 0);\n   matrix OUTPUT_MAT = transform(INPUT_MAT);\n   \n   return np.flatten(OUTPUT_MAT);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CNMF::fit_transform(matrix &X, uint k=2)\n {\n  ulong m = X.Rows(), n = X.Cols();\n  double best_frobenius_norm = DBL_MIN;\n  \n   m_components = m_components == 0 ? (uint)n : k;      \n   \n//--- Initialize Random values \n\n   np.random.seed(this.m_randseed);\n   this.W = np.random.randn((uint)m, this.m_components);  \n   this.H = np.random.randn(this.m_components, (uint)n);\n   \n//--- Update factors\n      \n   vector loss(this.m_max_iter);\n    for (uint i=0; i<this.m_max_iter; i++)\n      {\n        // Update W\n         this.W *= MathAbs((X.MatMul(this.H.Transpose())) / (this.W.MatMul(this.H.MatMul(this.H.Transpose()))+ 1e-10));\n         \n        // Update H\n         this.H *= MathAbs((this.W.Transpose().MatMul(X)) / (this.W.Transpose().MatMul(this.W.MatMul(this.H))+ 1e-10));\n         \n         loss[i] = MathPow((X - W.MatMul(H)).Flat(1), 2);\n                    \n         // Calculate Frobenius norm of the difference\n        double frobenius_norm = (X - W.MatMul(H)).Norm(MATRIX_NORM_FROBENIUS);\n\n         if (MQLInfoInteger(MQL_DEBUG))\n           printf(\"%s [%d/%d] Loss = %.5f frobenius norm %.5f\",__FUNCTION__,i+1,m_max_iter,loss[i],frobenius_norm);\n         \n          // Check convergence\n          if (frobenius_norm < this.m_tol)\n              break;\n      }\n  \n  return this.W.MatMul(this.H); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint CNMF::select_best_components(matrix &X)\n{\n    uint best_components = 1;\n    this.m_components = (uint)X.Cols();\n    \n    vector explained_ratio(X.Cols());    \n    for (uint k = 1; k <= X.Cols(); k++)\n    {\n       // Calculate explained variance or other criterion \n       matrix X_reduced = fit_transform(X, k);\n   \n       // Calculate explained variance as the ratio of squared Frobenius norms\n       double explained_variance = 1.0 - (X-X_reduced).Norm(MATRIX_NORM_FROBENIUS) / (X.Norm(MATRIX_NORM_FROBENIUS));\n        \n        if (MQLInfoInteger(MQL_DEBUG))\n            printf(\"k %d Explained Var %.5f\",k,explained_variance);\n       \n       explained_ratio[k-1] = explained_variance;       \n    }\n    \n    return uint(explained_ratio.ArgMax()+1);\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Decomposition/PCA.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                          pca.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//|         Principle Component Analysis Library                     |\n//+------------------------------------------------------------------+\n#include \"base.mqh\"\n#include <MALE5\\MqPlotLib\\plots.mqh>\n#include <MALE5\\Numpy\\Numpy.mqh>\n//+------------------------------------------------------------------+\n//|            Principal Component Analysis Class                    |\n//+------------------------------------------------------------------+\nclass CPCA\n  {\n\nenum criterion\n  {\n    CRITERION_VARIANCE,\n    CRITERION_KAISER,\n    CRITERION_SCREE_PLOT\n  };\n  \nCPlots   plt;\nCNumpy   np;\n\nprotected:\n   uint              m_components;\n   criterion         m_criterion;\n   \n   matrix            components_matrix;\n   vector            mean;   \n   \n   uint              n_features;\n                     \n                     \n                     uint extract_components(vector &eigen_values, double threshold=0.95);\n                     \npublic:\n                     CPCA(int k=0, criterion CRITERION_=CRITERION_SCREE_PLOT);\n                    ~CPCA(void);\n                    \n                     matrix fit_transform(matrix &X);\n                     matrix transform(matrix &X);\n                     vector transform(vector &X);\n                     bool save(string dir);\n                     bool load(string dir);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPCA::CPCA(int k=0, criterion CRITERION_=CRITERION_SCREE_PLOT)\n :m_components(k),\n  m_criterion(CRITERION_)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCPCA::~CPCA(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CPCA::fit_transform(matrix &X)\n {    \n   n_features = (uint)X.Cols();\n   \n   if (m_components>n_features)\n     {\n       printf(\"%s Number of dimensions K[%d] is supposed to be <= number of features %d\",__FUNCTION__,m_components,n_features);\n       this.m_components = (int)n_features;\n       Print(__LINE__);\n     }\n\n//---\n   \n   this.mean = X.Mean(0);\n   \n   matrix X_centered = BaseDimRed::subtract(X, this.mean);   \n   BaseDimRed::ReplaceNaN(X_centered);\n   \n   matrix cov_matrix = cova(X_centered, false);\n   \n   matrix eigen_vectors;\n   vector eigen_values;\n    \n   BaseDimRed::ReplaceNaN(cov_matrix);\n   \n   if (!cov_matrix.Eig(eigen_vectors, eigen_values))\n     printf(\"Failed to caculate Eigen matrix and vectors Err=%d\",GetLastError());\n   \n//--- Sort eigenvectors by decreasing eigenvalues\n   \n   vector args = np.argsort(eigen_values); \n   args = np.reverse(args);\n   \n   eigen_values = BaseDimRed::Sort(eigen_values, args);\n   eigen_vectors = BaseDimRed::Sort(eigen_vectors, args);\n   \n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n      Print(\"Eigen values: \",eigen_values);\n      \n   if (m_components==0)\n     m_components = this.extract_components(eigen_values);\n   else\n     this.extract_components(eigen_values);\n   \n   if (MQLInfoInteger(MQL_DEBUG)) \n     printf(\"%s Selected components %d\",__FUNCTION__,m_components);\n   \n   this.components_matrix = BaseDimRed::Slice(eigen_vectors, m_components, 1); //Get the components matrix\n   //MatrixExtend::NormalizeDouble_(this.components_matrix, 5);\n   //this.components_matrix = scaler.fit_transform(this.components_matrix.Transpose()); //Normalize components matrix\n   \n   this.components_matrix = this.components_matrix.Transpose();\n   \n   if (MQLInfoInteger(MQL_DEBUG))\n     Print(\"components_matrix\\n\",components_matrix);\n   \n//---\n      \n   return X_centered.MatMul(components_matrix.Transpose()); //return the pca scores\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CPCA::transform(matrix &X)\n {\n   if (X.Cols()!=this.n_features)\n     {\n       printf(\"%s Inconsistent input X matrix size, It is supposed to be of size %d same as the matrix used under fit_transform\",__FUNCTION__,n_features);\n       this.m_components = n_features;\n     }\n     \n   matrix X_centered = BaseDimRed::subtract(X, this.mean);\n\n   return X_centered.MatMul(this.components_matrix.Transpose()); //return the pca scores\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CPCA::transform(vector &X)\n {\n   matrix INPUT_MAT = np.expand_dims(X, 0);\n   matrix OUTPUT_MAT = transform(INPUT_MAT);\n      \n   return np.flatten(OUTPUT_MAT);\n }\n//+------------------------------------------------------------------+\n//|   Select the number of components based on some criterion        |\n//+------------------------------------------------------------------+\nuint CPCA::extract_components(vector &eigen_values, double threshold=0.95)\n {\n  uint k = 0;\n  \n   vector eigen_pow = MathPow(eigen_values, 2);\n   vector cum_sum = eigen_pow.CumSum();\n   double sum = eigen_pow.Sum();\n   \n   switch(m_criterion)\n     {\n      case  CRITERION_VARIANCE: \n         {              \n            \n            vector cumulative_variance =  cum_sum / sum;\n            \n            if (MQLInfoInteger(MQL_DEBUG))\n              Print(\"Cummulative variance: \",cumulative_variance);\n            \n            vector v(cumulative_variance.Size());  v.Fill(0.0);\n            for (ulong i=0; i<v.Size(); i++)\n              v[i] = (cumulative_variance[i] >= threshold);\n               \n            k = (uint)v.ArgMax() + 1;\n         }  \n         \n        break;\n        \n      case  CRITERION_KAISER:\n         {\n           vector v(eigen_values.Size()); v.Fill(0.0);\n            for (ulong i=0; i<eigen_values.Size(); i++)\n              v[i] = (eigen_values[i] >= 1);\n            \n            k = uint(v.Sum());\n         } \n        \n        break;\n        \n      case  CRITERION_SCREE_PLOT:\n       {  \n         vector v_cols(eigen_values.Size());\n         \n         for (ulong i=0; i<v_cols.Size(); i++)\n             v_cols[i] = (int)i+1;\n             \n          vector vars = eigen_values;\n          \n          //matrix_utils.Sort(vars); //Make sure they are in ascending first order\n          //matrix_utils.Reverse(vars);  //Set them to descending order\n          \n          plt.Plot(\"Scree plot\",v_cols,vars,\"EigenValue\",\"Principal Components\",\"EigenValue\",CURVE_POINTS_AND_LINES);\n\n//---\n      string warn = \"\\n<<<< WARNING >>>>\\nThe Scree plot doesn't return the determined number of k components\\nThe cummulative variance will return the number of k components instead\\nThe k returned might be different from what you see on the scree plot\";\n             warn += \"\\nTo apply the same number of k components to the PCA from the scree plot\\nCall the PCA model again with that value applied from the plot\\n\";\n      \n         Print(warn);\n        \n        //--- Kaiser\n        \n           vector v(eigen_values.Size()); v.Fill(0.0);\n            for (ulong i=0; i<eigen_values.Size(); i++)\n              v[i] = (eigen_values[i] >= 1);\n            \n            k = uint(v.Sum());\n        }          \n           \n        break;\n     } \n     \n   return (k);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CPCA::save(string dir)\n { \n   \n   matrix m = CUtils::VectorToMatrix(this.mean, this.mean.Size());\n   \n   if (!CUtils::WriteCsv(dir+\"\\\\PCA-Mean.csv\",m,NULL,false,8))\n     {\n       Print(\"Failed to Save PCA-Mean information to \",dir);\n       return false;\n     }\n\n//---\n\n   if (!CUtils::WriteCsv(dir+\"\\\\PCA-ComponentsMatrix.csv\",this.components_matrix,NULL,false,8))\n     {\n       Print(\"Failed to Save PCA-ComponentsMatrix information to \",dir);\n       return false;\n     }\n     \n   return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CPCA::load(string dir)\n {\n   string header;\n   matrix m = CUtils::ReadCsv(dir+\"\\\\PCA-Mean.csv\",header);\n   \n   if (m.Rows()==0)\n     return false;\n     \n   this.mean = CUtils::MatrixToVector(m);\n   this.n_features = (uint)this.mean.Size();\n   \n//---\n   \n   this.components_matrix = CUtils::ReadCsv(dir+\"\\\\PCA-ComponentsMatrix.csv\",header);\n   \n   //printf(\"Components Matrix[%dx%d]\",components_matrix.Rows(),components_matrix.Cols());\n   \n   if (components_matrix.Rows()==0)\n     return false;\n     \n   return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nmatrix cova(matrix &data, bool row_var=true)\n  {\n    if (row_var)\n        data = data.Transpose();  // Transpose if each row represents a data point\n\n    // Step 1: Center the data\n    matrix centered_data = BaseDimRed::subtract(data, data.Mean(0));\n\n    // Step 2: Calculate the covariance matrix\n    matrix covariance_matrix = centered_data.Transpose().MatMul(centered_data) / (data.Rows() - 1);\n\n    return covariance_matrix;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Sklearn/Decomposition/README.md",
    "content": "## Linear Discriminant Analysis (LDA) \n\nThis documentation explains the `CLDA` class in MQL5, which implements **Linear Discriminant Analysis (LDA)** for dimensionality reduction and classification tasks.\n\n**I. LDA Theory:**\n\nLDA is a supervised learning technique that aims to find **linear projections** of the data that **maximize the separation between different classes** while **minimizing variance within each class**. This makes it particularly useful for **classification** problems where the goal is to distinguish between distinct groups.\n\n**II. CLDA Class:**\n\nThe `CLDA` class provides functionalities for performing LDA in MQL5:\n\n**Public Functions:**\n\n* `CLDA(uint k=NULL, lda_criterion CRITERION_=CRITERION_SCREE_PLOT, double reg_param =1e-6)` Constructor, allows setting hyperparameters:\n    * `k`: Number of components to extract (default: None, determined automatically).\n    * `CRITERION_`: Criterion for selecting the best components (default: CRITERION_SCREE_PLOT).\n    * `reg_param`: Regularization parameter to prevent overfitting (default: 1e-6).\n* `~CLDA(void)`: Destructor.\n* `matrix fit_transform(const matrix &x, const vector &y):` Trains the model on the provided data (`x` - independent variables, `y` - class labels) and returns the transformed data.\n* `matrix transform(const matrix &x):` Transforms new data (`x`) using the trained model.\n* `vector transform(const vector &x):` Transforms a single new data point (`x`) using the trained model.\n\n**Internal Functions:**\n\n* `calculate_variance(vector &eigen_values, double threshold=0.95)`: Calculates the number of components based on explained variance (optional).\n* `calculate_kaiser(vector &eigen_values)`: Calculates the number of components based on Kaiser's criterion (optional).\n* `extract_components(vector &eigen_values, double threshold=0.95)`: Extracts the selected number of components based on the chosen criterion.\n\n**III. Additional Notes:**\n\n* The `m_criterion` member variable allows choosing the criterion for selecting the number of components:\n    * **CRITERION_VARIANCE:** Retains components that explain a specific percentage of variance (set by `threshold`).\n    * **CRITERION_KAISER:** Retains components with eigenvalues greater than 1.\n    * **CRITERION_SCREE_PLOT:** Analyzes the scree plot to visually determine the number of components with significant eigenvalues.\n* The `m_regparam` member variable allows for regularization to prevent overfitting.\n* The class internally uses the `CPlots` class (not documented here) for potential visualization purposes (e.g., scree plot).\n\n\n\n## Principal Component Analysis (PCA) \n\nThis documentation explains the `CPCA` class in MQL5, which implements **Principal Component Analysis (PCA)** for dimensionality reduction and data visualization tasks.\n\n**I. PCA Theory:**\n\nPCA is an unsupervised learning technique that aims to find a **linear transformation** of the data that captures the most **variance** in a **reduced number of dimensions**. This allows for:\n\n* **Dimensionality reduction:** Reduce the number of features while retaining most of the information in the data.\n* **Data visualization:** Project high-dimensional data onto a lower-dimensional space for easier visualization.\n\n**II. CPCA Class:**\n\nThe `CPCA` class provides functionalities for performing PCA in MQL5:\n\n**Public Functions:**\n\n* `CPCA(int k=0, criterion CRITERION_=CRITERION_SCREE_PLOT)` Constructor, allows setting hyperparameters:\n    * `k`: Number of components to extract (default: 0, determined automatically using the chosen criterion).\n    * `CRITERION_`: Criterion for selecting the best components (default: CRITERION_SCREE_PLOT).\n* `~CPCA(void)` Destructor.\n* `matrix fit_transform(matrix &X)` Trains the model on the provided data (`X`) and returns the transformed data.\n* `matrix transform(matrix &X)` Transforms new data (`X`) using the trained model.\n* `vector transform(vector &X)` Transforms a single new data point (`X`) using the trained model.\n* `bool save(string dir)` Saves the model parameters to a specified directory (`dir`).\n* `bool load(string dir)` Loads the model parameters from a specified directory (`dir`).\n\n**Internal Functions:**\n\n* `extract_components(vector &eigen_values, double threshold=0.95)`: Extracts the selected number of components based on the chosen criterion (similar to the `CLDA` class).\n\n**III. Additional Notes:**\n\n* The `m_criterion` member variable allows choosing the criterion for selecting the number of components (same options as in `CLDA`).\n* The class internally uses the `CPlots` class (not documented here) for potential visualization purposes.\n* Saving and loading functionalities allow for model persistence and reusability.\n\n\n\n## Non-Negative Matrix Factorization (NMF) \n\nThis documentation explains the `CNMF` class in MQL5, which implements **Non-Negative Matrix Factorization (NMF)** for data decomposition and feature extraction tasks.\n\n**I. NMF Theory:**\n\nNMF is a dimensionality reduction technique that decomposes a **non-negative matrix** `V` into two **non-negative matrices** `W` and `H`:\n\n* `V` (shape: `m x n`): The input data matrix, where `m` is the number of data points and `n` is the number of features.\n* `W` (shape: `m x k`): The **basis matrix**, where `k` is the number of chosen components and each row represents a **basis vector**.\n* `H` (shape: `k x n`): The **coefficient matrix**, where each row corresponds to a basis vector and each element represents the **contribution** of that basis vector to a specific feature in the original data.\n\nBy finding a suitable factorization, NMF aims to represent the original data as a **linear combination** of basis vectors while preserving the non-negative nature of the input data. This allows for:\n\n* **Dimensionality reduction:** Reduce the number of features while still capturing essential information.\n* **Feature extraction:** Identify underlying factors or patterns in the data through the basis vectors.\n* **Data interpretation:** Gain insights into the data by analyzing the non-negative contributions of basis vectors to each feature.\n\n**II. CNMF Class:**\n\nThe `CNMF` class provides functionalities for performing NMF :\n\n**Public Functions:**\n\n* `CNMF(uint max_iter=100, double tol=1e-4, int random_state=-1)` Constructor, allows setting hyperparameters:\n    * `max_iter`: Maximum number of iterations for the NMF algorithm (default: 100).\n    * `tol`: Tolerance for convergence (default: 1e-4).\n    * `random_state`: Random seed for initialization (default: -1, uses random seed).\n* `~CNMF(void)` Destructor.\n* `matrix fit_transform(matrix &X, uint k=2)` Trains the model on the provided data (`X`) and returns the decomposed components (`W` and `H`).\n    * `k`: Number of components to extract (default: 2).\n* `matrix transform(matrix &X)` Transforms new data (`X`) using the trained model.\n* `vector transform(vector &X)` Transforms a single new data point (`X`) using the trained model.\n* `uint select_best_components(matrix &X)` Analyzes the input data and suggests an appropriate number of components (implementation details might vary depending on the specific NMF algorithm used).\n\n**III. Additional Notes:**\n\n* The internal implementation details of the NMF algorithm might vary depending on the specific chosen library or technique.\n* Choosing the appropriate number of components is crucial for optimal performance and avoiding overfitting. The `select_best_components` function is helpful as a starting point, but further evaluation and domain knowledge might be needed for optimal selection.\n\nBy understanding the theoretical foundation and functionalities of the `CNMF` class, MQL5 users can leverage NMF for various tasks, including:\n\n* Dimensionality reduction for data visualization or machine learning algorithms that require lower-dimensional inputs.\n* Feature extraction to identify underlying structure or patterns in non-negative data.\n* Topic modeling for analyzing text data or other types of document collections.\n\n\n\n## Truncated Singular Value Decomposition (Truncated SVD) \n\nThis documentation explains the `CTruncatedSVD` class in MQL5, which implements **Truncated Singular Value Decomposition (Truncated SVD)** for dimensionality reduction and data visualization tasks.\n\n**I. Truncated SVD Theory:**\n\nTruncated SVD is a dimensionality reduction technique based on **Singular Value Decomposition (SVD)**. SVD decomposes a matrix `X` into three matrices:\n\n* `U`: A left singular vectors matrix.\n* `Σ`: A diagonal matrix containing the singular values of `X`.\n* `V^T`: A right singular vectors matrix (transposed).\n\nTruncated SVD retains only **k** top singular values from `Σ` and their corresponding columns from `U` and `V^T`. This creates a lower-dimensional representation of the original data that captures most of the variance.\n\n**II. CTruncatedSVD Class:**\n\nThe `CTruncatedSVD` class provides functionalities for performing Truncated SVD in MQL5:\n\n**Public Functions:**\n\n* `CTruncatedSVD(uint k=0):` Constructor, allows setting the number of components (`k`) to retain (default: 0, determined automatically).\n* `~CTruncatedSVD(void):` Destructor.\n* `matrix fit_transform(matrix& X):` Trains the model on the provided data (`X`) and returns the transformed data.\n* `matrix transform(matrix &X):` Transforms new data (`X`) using the trained model.\n* `vector transform(vector &X):` Transforms a single new data point (`X`) using the trained model.\n* `ulong _select_n_components(vector &singular_values):` (Internal function, not directly exposed to users) Determines the number of components based on the explained variance ratio (implementation details may vary).\n\n**III. Additional Notes:**\n\n* The class internally uses the `CPlots` class (not documented here) for potential visualization purposes.\n* Choosing the appropriate number of components (`k`) is crucial for balancing dimensionality reduction and information preservation. The `_select_n_components` function might use different criteria (e.g., explained variance ratio) for automatic selection, and user discretion might be needed depending on the specific task.\n\nBy understanding the theoretical foundation and functionalities of the `CTruncatedSVD` class, MQL5 users can leverage Truncated SVD for:\n\n* **Dimensionality reduction:** Reduce the number of features while retaining most of the information.\n* **Data visualization:** Project high-dimensional data onto a lower-dimensional space for easier visualization.\n* **Feature extraction:** Identify underlying factors or patterns in the data through the singular vectors (although not explicitly mentioned in the class functionalities).\n\nIt's important to note that the specific implementation details of the `CTruncatedSVD` class and the `_select_n_components` function might vary depending on the chosen MQL5 library or framework. Refer to the specific documentation of your chosen library for the most accurate and up-to-date information.\n\n\n**Reference:**\n* [Data Science and Machine Learning (Part 18): The battle of Mastering Market Complexity, Truncated SVD Versus NMF](https://www.mql5.com/en/articles/13968)\n* [Data Science and Machine Learning(Part 20) : Algorithmic Trading Insights, A Faceoff Between LDA and PCA in MQL5](https://www.mql5.com/en/articles/14128)\n"
  },
  {
    "path": "Sklearn/Decomposition/TruncatedSVD.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                 CTruncatedSVD.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include \"base.mqh\"\n#include <MALE5\\MqPlotLib\\plots.mqh>\n\nclass CTruncatedSVD\n  {\nCPlots   plt;\n\nuint m_components;\nulong n_features;\nmatrix components_;\nvector mean;\nvector explained_variance_;\n\npublic:\n                     CTruncatedSVD(uint k=0);\n                    ~CTruncatedSVD(void);\n                    \n                    matrix fit_transform(matrix& X);\n                    matrix transform(matrix &X);\n                    vector transform(vector &X);\n                    ulong _select_n_components(vector &singular_values);\n  };\n//+------------------------------------------------------------------+\n//|  Once the k value is left to default value of zero, the function |\n//| _select_n_components will be used to find the best number of     |\n//| components to use                                                |\n//+------------------------------------------------------------------+\nCTruncatedSVD::CTruncatedSVD(uint k=0)\n:m_components(k)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCTruncatedSVD::~CTruncatedSVD(void)\n {\n \n } \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CTruncatedSVD::fit_transform(matrix &X)\n {\n  n_features = X.Cols();  \n    \n   if (m_components>n_features)\n     {\n       printf(\"%s Number of dimensions K[%d] is supposed to be <= number of features %d\",__FUNCTION__,m_components,n_features);\n       this.m_components = (uint)n_features;\n     }\n     \n//--- Center the data (subtract mean)\n\n    this.mean = X.Mean(0);\n    matrix X_centered = BaseDimRed::subtract(X, this.mean);\n    \n//--- Compute the covariance matrix\n   \n    BaseDimRed::ReplaceNaN(X_centered);\n    matrix cov_matrix = X_centered.Cov(false);\n    \n//---  Perform SVD on the covariance matrix\n\n    matrix U={}, Vt={};\n    vector Sigma={};\n    \n    BaseDimRed::ReplaceNaN(cov_matrix);\n    \n    if (!cov_matrix.SVD(U,Vt,Sigma))\n       Print(__FUNCTION__,\" Line \",__LINE__,\" Failed to calculate SVD Err=\",GetLastError());    \n        \n     if (m_components == 0)\n       {\n         m_components = (uint)this._select_n_components(Sigma);\n         Print(__FUNCTION__,\" Best value of K = \",m_components);\n       }\n                 \n    this.components_ = BaseDimRed::Slice(Vt, this.m_components).Transpose();\n    BaseDimRed::ReplaceNaN(this.components_);\n        \n    if (MQLInfoInteger(MQL_DEBUG))\n      Print(\"components_T[\",components_.Rows(),\"X\",components_.Cols(),\"]\\n\",this.components_);\n    \n    this.explained_variance_ = MathPow(BaseDimRed::Slice(Sigma, this.m_components), 2) / (X.Rows() - 1);\n    \n    return X_centered.MatMul(components_);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CTruncatedSVD::transform(matrix &X)\n {\n   matrix X_centered = BaseDimRed::subtract(X, this.mean);\n   \n   if (X.Cols()!=this.n_features)\n     {\n       printf(\"%s Inconsistent input X matrix size, It is supposed to be of size %d same as the matrix used under fit_transform\",__FUNCTION__,n_features);\n       this.m_components = (uint)n_features;\n     }\n    \n    return X_centered.MatMul(components_);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CTruncatedSVD::transform(vector &X)\n {\n   matrix INPUT_MAT = MatrixExtend::VectorToMatrix(X, X.Size());\n   matrix OUTPUT_MAT = transform(INPUT_MAT);\n      \n   return MatrixExtend::MatrixToVector(OUTPUT_MAT);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nulong CTruncatedSVD::_select_n_components(vector &singular_values)\n {\n    double total_variance = MathPow(singular_values.Sum(), 2);\n    \n    vector explained_variance_ratio = MathPow(singular_values, 2).CumSum() / total_variance;\n    \n    if (MQLInfoInteger(MQL_DEBUG))\n      Print(__FUNCTION__,\" Explained variance ratio \",explained_variance_ratio);\n    \n    vector k(explained_variance_ratio.Size());\n    \n    for (uint i=0; i<k.Size(); i++)\n      k[i] = i+1;\n    \n    plt.Plot(\"Explained variance plot\",k,explained_variance_ratio,\"variance\",\"components\",\"Variance\");\n    \n   return explained_variance_ratio.ArgMax() + 1;  //Choose k for maximum explained variance\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Decomposition/base.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                      helpers.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| BaseDimRed class for dimension reduction, containing most useful       |\n//| that are necessary for the algorithms in this folder             |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\Utils.mqh>\n\nclass BaseDimRed\n  {\npublic:\n                     BaseDimRed(void);\n                    ~BaseDimRed(void);\n                    \n                    static matrix Slice(const matrix &mat, uint from_0_to, int axis=0);\n                    static vector Slice(const vector &v, uint from_0_to);\n                    static matrix subtract(const matrix&mat, const vector &v);\n                    static void   ReplaceNaN(matrix &mat);\n                    static matrix Sort(matrix &mat, vector &args);\n                    static vector Sort(vector &v, vector &args);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix BaseDimRed::Slice(const matrix &mat, uint from_0_to, int axis=0)\n {\n  matrix ret = {};\n  \n  if (from_0_to==0)\n    {\n      printf(\"%s Cannot slice a vector from index 0 to 0\",__FUNCTION__);\n      return mat;\n    }\n  \n  switch(axis)\n    {\n     case  0: \n      ret.Resize(from_0_to, mat.Cols());\n       \n      for (uint i=0; i<mat.Rows(); i++)\n        ret.Row(mat.Row(i), i);   \n        \n       break;\n     case 1:\n      ret.Resize(mat.Rows(), from_0_to);\n      \n      for (uint i=0; i<mat.Cols(); i++)\n        ret.Col(mat.Col(i), i);   \n        \n       break;\n     default:\n       Print(\"%s Invalid axis %d axis can be either 0 or 1\",__FUNCTION__,axis);\n       break;\n   }\n   \n   return ret;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector BaseDimRed::Slice(const vector &v, uint from_0_to)\n {\n   vector ret(from_0_to);\n   \n  if (from_0_to==0)\n    {\n      printf(\"%s Cannot slice a vector from index 0 to 0\",__FUNCTION__);\n      return v;\n    }\n    \n   for (uint i=0; i<ret.Size(); i++)\n     ret[i] = v[i];\n  \n   return ret;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix BaseDimRed::subtract(const matrix&mat, const vector &v)\n {\n   matrix ret = mat;\n   \n   if (mat.Rows()!=v.Size() && mat.Cols()!=v.Size())\n     {\n       printf(\"%s Dimensions Mismatch\",__FUNCTION__);\n       matrix empty = {};\n       return empty;\n     }\n   \n   bool isrows = v.Size()==mat.Rows() ? true : false;\n   \n   for (ulong i=0; i<(isrows?mat.Cols():mat.Rows()); i++)\n    {\n      if (isrows)\n       ret.Col(mat.Col(i)-v, i);\n      else\n       ret.Row(mat.Row(i)-v, i);\n    }\n   \n   return ret; \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid BaseDimRed::ReplaceNaN(matrix &mat)\n {\n   for (ulong i = 0; i < mat.Rows(); i++) \n     for (ulong j = 0; j < mat.Cols(); j++) \n       if (!MathIsValidNumber(mat[i][j]))\n          mat[i][j] = 0.0;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix BaseDimRed::Sort(matrix &mat, vector &args)\n {\n   matrix m = mat;\n   \n   if (args.Size() != mat.Cols())\n     {\n       printf(\"%s Args size != mat.Cols \");\n       return m;\n     }\n   \n   for (ulong i=0; i<mat.Cols(); i++)\n       m.Col(mat.Col((ulong)args[i]), i);\n       \n   return m;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector BaseDimRed::Sort(vector &v, vector &args)\n {\n   vector vec = v;\n   \n   if (args.Size() != v.Size())\n     {\n       printf(\"%s Args size != v.size \");\n       return vec;\n     }\n   \n   for (ulong i=0; i<v.Size(); i++)\n     vec[i] = v[(int)args[i]];\n       \n   return vec;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Ensemble/AdaBoost.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                     AdaBoost.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\MatrixExtend.mqh>\n#include <MALE5\\Naive Bayes\\Naive Bayes.mqh>\n\n//+------------------------------------------------------------------+\n//|      Model class                                                 |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\Linear Models\\Logistic Regression.mqh>\n\n//+------------------------------------------------------------------+\n//|         AdaBoost class for Decision Tree                         |\n//+------------------------------------------------------------------+\nnamespace DecisionTree\n {\nclass AdaBoost\n  {\n  \nprotected:\n                     vector m_alphas;\n                     vector classes_in_data;\n                     int m_random_state;\n                     bool m_boostrapping;\n                     uint m_min_split, m_max_depth;\n                     \n                     CDecisionTreeClassifier *weak_learners[]; //store weak_learner pointers for memory allocation tracking\n                     CDecisionTreeClassifier *weak_learner;\n                     \n                     uint m_estimators;\n                     \npublic:\n                     AdaBoost(uint min_split, uint max_split, uint n_estimators=50, int random_state=42, bool bootstrapping=true);\n                    ~AdaBoost(void);\n                    \n                    void fit(matrix &x, vector &y);\n                    int predict(vector &x);\n                    vector predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nAdaBoost::AdaBoost(uint min_split, uint max_split, uint n_estimators=50, int random_state=42, bool bootstrapping=true)\n:m_estimators(n_estimators),\n m_random_state(random_state),\n m_boostrapping(bootstrapping),\n m_min_split(min_split),\n m_max_depth(max_split)\n {\n   ArrayResize(weak_learners, m_estimators);   //Resizing the array to retain the number of base weak_learners\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nAdaBoost::~AdaBoost(void)\n {   \n   for (uint i=0; i<m_estimators; i++) //Delete the forest | all trees\n     if (CheckPointer(weak_learners[i]) != POINTER_INVALID)\n      delete(weak_learners[i]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid AdaBoost::fit(matrix &x,vector &y)\n {\n   m_alphas.Resize(m_estimators);  \n   classes_in_data = MatrixExtend::Unique(y); //Find the target variables in the class\n      \n   ulong m = x.Rows(), n = x.Cols();\n   vector weights(m); weights = weights.Fill(1.0) / m; //Initialize instance weights\n   vector preds(m);\n   vector misclassified(m);\n   \n//---\n\n   matrix data = MatrixExtend::concatenate(x, y);\n   matrix temp_data;\n   \n   matrix x_subset;\n   vector y_subset;\n\n   double error = 0;\n   \n   for (uint i=0; i<m_estimators; i++)\n    {      \n      temp_data = data;\n      MatrixExtend::Randomize(temp_data, this.m_random_state, this.m_boostrapping);\n      \n       if (!MatrixExtend::XandYSplitMatrices(temp_data, x_subset, y_subset)) //Get randomized subsets\n         {  \n            ArrayRemove(weak_learners,i,1); //Delete the invalid weak_learner\n            printf(\"%s %d Failed to split data\",__FUNCTION__,__LINE__);\n            continue;\n         }\n\n//---\n      \n      weak_learner = new CDecisionTreeClassifier(this.m_min_split, m_max_depth);             \n              \n      weak_learner.fit(x_subset, y_subset); //fiting the randomized data to the i-th weak_learner\n      preds = weak_learner.predict_bin(x_subset); //making predictions for the i-th weak_learner\n             \n       for (ulong j=0; j<m; j++)\n          misclassified[j] = (preds[j] != y_subset[j]);\n       \n       error = (misclassified * weights).Sum() / (double)weights.Sum();\n       \n      //--- Calculate the weight of a weak learner in the final weak_learner\n      \n      double alpha = 0.5 * log((1-error) / (error + 1e-10));\n      \n      //--- Update instance weights\n      \n      weights *= exp(-alpha * y_subset * preds);\n      weights /= weights.Sum();\n      \n      //--- save a weak learner and its weight\n      \n      this.m_alphas[i] = alpha;\n      this.weak_learners[i] = weak_learner;\n      \n      printf(\"Building Estimator [%d/%d] Accuracy Score %.3f\",i+1,m_estimators,Metrics::accuracy_score(y_subset,preds));\n    }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint AdaBoost::predict(vector &x)\n {\n   // Combine weak learners using weighted sum   \n   \n   vector weak_preds(m_estimators), \n          final_preds(m_estimators);\n          \n   for (uint i=0; i<this.m_estimators; i++)\n     weak_preds[i] = this.weak_learners[i].predict_bin(x);\n  \n  return (int)weak_preds[(this.m_alphas*weak_preds).ArgMax()]; //Majority decision\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector AdaBoost::predict(matrix &x)\n {\n   vector ret_v(x.Rows());\n   for (ulong i=0; i<ret_v.Size(); i++)\n      ret_v[i] = this.predict(x.Row(i));\n   \n   return ret_v;\n }\n}\n\n//+------------------------------------------------------------------+\n//|         Adaboost for Logistic Regression                         |\n//+------------------------------------------------------------------+\n\nnamespace LogisticRegression\n {\nclass AdaBoost\n  {\n  \nprotected:\n                     vector m_alphas;\n                     vector classes_in_data;\n                     int m_random_state;\n                     bool m_boostrapping;\n                     uint m_min_split, m_max_depth;\n                     \n                     CLogisticRegression *weak_learners[]; //store weak_learner pointers for memory allocation tracking\n                     CLogisticRegression *weak_learner;\n                     \n                     uint m_estimators;\n                     \npublic:\n                     AdaBoost(uint n_estimators=50, int random_state=42, bool bootstrapping=true);\n                    ~AdaBoost(void);\n                    \n                    void fit(matrix &x, vector &y);\n                    int predict(vector &x);\n                    vector predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nAdaBoost::AdaBoost(uint n_estimators=50, int random_state=42, bool bootstrapping=true)\n:m_estimators(n_estimators),\n m_random_state(random_state),\n m_boostrapping(bootstrapping)\n {\n   ArrayResize(weak_learners, m_estimators);   //Resizing the array to retain the number of base weak_learners\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nAdaBoost::~AdaBoost(void)\n {   \n   for (uint i=0; i<m_estimators; i++) //Delete the forest | all trees\n     if (CheckPointer(weak_learners[i]) != POINTER_INVALID)\n      delete(weak_learners[i]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid AdaBoost::fit(matrix &x,vector &y)\n {\n   m_alphas.Resize(m_estimators);  \n   classes_in_data = MatrixExtend::Unique(y); //Find the target variables in the class\n      \n   ulong m = x.Rows(), n = x.Cols();\n   vector weights(m); weights = weights.Fill(1.0) / m; //Initialize instance weights\n   vector preds(m);\n   vector misclassified(m);\n   \n//---\n\n   matrix data = MatrixExtend::concatenate(x, y);\n   matrix temp_data;\n   \n   matrix x_subset;\n   vector y_subset;\n\n   double error = 0;\n   \n   for (uint i=0; i<m_estimators; i++)\n    {      \n      \n      temp_data = data;\n      MatrixExtend::Randomize(temp_data, this.m_random_state, this.m_boostrapping);\n      \n       if (!MatrixExtend::XandYSplitMatrices(temp_data, x_subset, y_subset)) //Get randomized subsets\n         {  \n            ArrayRemove(weak_learners,i,1); //Delete the invalid weak_learner\n            printf(\"%s %d Failed to split data\",__FUNCTION__,__LINE__);\n            continue;\n         }\n\n//---\n      \n      weak_learner = new CLogisticRegression();             \n              \n      weak_learner.fit(x_subset, y_subset); //fiting the randomized data to the i-th weak_learner\n      preds = weak_learner.predict(x_subset); //making predictions for the i-th weak_learner\n             \n       for (ulong j=0; j<m; j++)\n          misclassified[j] = (preds[j] != y_subset[j]);\n       \n       error = (misclassified * weights).Sum() / (double)weights.Sum();\n       \n      //--- Calculate the weight of a weak learner in the final weak_learner\n      \n      double alpha = 0.5 * log((1-error) / (error + 1e-10));\n      \n      //--- Update instance weights\n      \n      weights *= exp(-alpha * y_subset * preds);\n      weights /= weights.Sum();\n      \n      //--- save a weak learner and its weight\n      \n      this.m_alphas[i] = alpha;\n      this.weak_learners[i] = weak_learner;\n      \n      printf(\"Building Estimator [%d/%d] Accuracy Score %.3f\",i+1,m_estimators,Metrics::accuracy_score(y_subset,preds));\n    }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint AdaBoost::predict(vector &x)\n {\n   // Combine weak learners using weighted sum   \n   \n   vector weak_preds(m_estimators), \n          final_preds(m_estimators);\n          \n   for (uint i=0; i<this.m_estimators; i++)\n     weak_preds[i] = this.weak_learners[i].predict(x);\n  \n  return (int)weak_preds[(this.m_alphas*weak_preds).ArgMax()]; //Majority decision\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector AdaBoost::predict(matrix &x)\n {\n   vector ret_v(x.Rows());\n   for (ulong i=0; i<ret_v.Size(); i++)\n      ret_v[i] = this.predict(x.Row(i));\n   \n   return ret_v;\n }\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Ensemble/README.md",
    "content": "## AdaBoost Ensemble Learning\n\nThis explanation covers the concept of AdaBoost and its implementation in `adaboost.mqh` for MQL5, highlighting the flexibility of using different weak learners (like decision trees or logistic regression).\n\n**I. AdaBoost Theory (Ensemble Learning Approach)`\n\nAdaBoost, short for Adaptive Boosting, is an ensemble learning algorithm that combines multiple **weak learners** (models with moderate predictive power) into a **strong learner** (model with improved predictive performance). It achieves this by:\n\n1. **Initializing weights for each data point:** Initially, all data points have equal weight.\n2. **Iteratively training weak learners:**\n    * In each iteration, a weak learner is trained on a **modified** dataset:\n        * If the previous learner misclassified a point, its weight is increased.\n        * If it was classified correctly, the weight is decreased. This focuses the subsequent learners on the \"harder\" data points.\n    * The weight of the current weak learner is determined based on its performance on the weighted data.\n3. **Combining the weak learners:**\n    * The final prediction of the ensemble is made by taking a weighted majority vote (classification) or a weighted average (regression) of the individual weak learner predictions, with higher weights given to more accurate learners.\n\n**II. AdaBoost.mqh Documentation:**\n\nThe `AdaBoost` class provides functionalities for implementing the AdaBoost algorithm using either **decision trees** or **logistic regression** as weak learners.\n\n**A. Common functionalities (present in both DecisionTree and LogisticRegression namespaces)`\n\n* `AdaBoost(uint n_estimators=50, int random_state=42, bool bootstrapping=true)` Constructor, allows setting hyperparameters (number of weak learners, random state for reproducibility, and enabling/disabling bootstrapping during training).\n* `~AdaBoost(void)` Destructor.\n* `void fit(matrix &x, vector &y):` Trains the ensemble model using the provided data (`x` - independent variables, `y` - dependent variables).\n* `int predict(vector &x):` Predicts the class label (for classification) for a new data point (`x`).\n* `vector predict(matrix &x):` Predicts class labels (for classification) for multiple new data points (`x`).\n\n**B. Namespace-specific functionalities:**\n\n* **DecisionTree namespace:**\n    * `CDecisionTreeClassifier *weak_learners[];`: Stores weak learner pointers (decision trees) for memory management.\n    * `CDecisionTreeClassifier *weak_learner;`: Internal pointer to the currently trained weak learner.\n* **LogisticRegression namespace:**\n    * `CLogisticRegression *weak_learners[];`: Stores weak learner pointers (logistic regression models) for memory management.\n    * `CLogisticRegression *weak_learner;`: Internal pointer to the currently trained weak learner.\n\n**III. Flexibility of Weak Learners:**\n\nThe key takeaway here is that the `AdaBoost` class is **not limited to** using decision trees as weak learners. The provided examples showcase its usage with both decision trees and logistic regression. This demonstrates the flexibility of the AdaBoost framework, where any model capable of making predictions (classification or regression) can be used as a weak learner.\n\n\n## Random Forest Classification and Regression: \n\nThis explanation covers the `CRandomForestClassifier` and `CRandomForestRegressor` classes in MQL5, which implement **random forests** for classification and regression tasks, respectively.\n\n**I. Random Forest Theory (Ensemble Learning Approach)`\n\nA random forest is an ensemble learning method that combines multiple **decision trees** into a single model to improve predictive performance. Each decision tree is trained on a **random subset of features** (independent variables) and a **bootstrapped sample** of the data (randomly drawn with replacement, increasing the importance of potentially informative data points). Predictions from all trees are then aggregated through **majority vote** (classification) or **averaging** (regression) to make the final prediction. This process reduces the variance of the model and helps prevent overfitting.\n\n**II. CRandomForestClassifier Class:**\n\nThis class provides functionalities for implementing a random forest for **classification** tasks.\n\n**Public Functions:**\n\n* `CRandomForestClassifier(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1)` Constructor, allows setting hyperparameters (number of trees, minimum samples per split, maximum tree depth, and random state for reproducibility).\n* `~CRandomForestClassifier(void)` Destructor.\n* `void fit(matrix &x, vector &y, bool replace=true, errors_classifier err=ERR_ACCURACY)` Trains the model on the provided data (`x` - independent variables, `y` - class labels).\n    * `replace` controls whether bootstrapping samples with replacement (True) or not (False).\n    * `err` specifies the error metric to use for internal training evaluation (default: ERR_ACCURACY).\n* `double predict(vector &x)` Predicts the class label for a new data point (`x`).\n* `vector predict(matrix &x)` Predicts class labels for multiple new data points (`x`).\n\n**Internal Functions:**\n\n* `ConvertTime(double seconds)`: Converts seconds to a human-readable format (not relevant for core functionality).\n* `err_metric(errors_classifier err, vector &actual, vector &preds)`: Calculates the specified error metric (e.g., accuracy) on given data (not directly exposed to users).\n\n**III. CRandomForestRegressor Class:**\n\nThis class implements a random forest for **regression** tasks. It inherits from `CRandomForestClassifier` and overrides specific functions for regression-specific behavior.\n\n**Public Functions:**\n\n* `CRandomForestRegressor(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1)` Constructor (same as for the classifier).\n* `~CRandomForestRegressor(void)` Destructor (same as for the classifier).\n* `void fit(matrix &x, vector &y, bool replace=true, errors_regressor err=ERR_R2_SCORE)` Trains the model (same as for the classifier, but default error metric is ERR_R2_SCORE).\n* `double predict(vector &x)` Predicts the continuous value for a new data point (`x`).\n* `vector predict(matrix &x)` Predicts continuous values for multiple new data points (`x`).\n\n**Internal Functions:**\n\n* Same as in `CRandomForestClassifier`.\n\n**IV. Key Points:**\n\n* Both classes use decision trees as base learners to build the random forest.\n* Hyperparameter tuning (number of trees, minimum samples per split, maximum depth) can significantly impact performance.\n* Random forests offer improved generalization and reduced variance compared to single decision trees.\n\n\n**Reference**\n* [Data Science and Machine Learning (Part 17): Money in the Trees? The Art and Science of Random Forests in Forex Trading](https://www.mql5.com/en/articles/13765)\n* [Data Science and Machine Learning (Part 19): Supercharge Your AI models with AdaBoost](https://www.mql5.com/en/articles/14034)"
  },
  {
    "path": "Sklearn/Ensemble/Random Forest.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                       forest.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include <MALE5\\Decision Tree\\tree.mqh>\n#include <MALE5\\metrics.mqh>\n\nenum errors_classifier\n  {\n   ERR_ACCURACY\n  };\n  \nclass CRandomForestClassifier\n  {\n\nprotected:\n   uint  m_ntrees;\n   uint  m_maxdepth;\n   uint  m_minsplit;\n   int   m_random_state;\n   \n   CDecisionTreeClassifier *forest[];\n   string ConvertTime(double seconds);\n   double err_metric(errors_classifier err, vector &actual, vector &preds);\n   \npublic:\n                     CRandomForestClassifier(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1);\n                    ~CRandomForestClassifier(void);\n                    \n                    void fit(matrix &x, vector &y, bool replace=true, errors_classifier err=ERR_ACCURACY);\n                    double predict(vector &x);\n                    vector predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRandomForestClassifier::CRandomForestClassifier(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1):\n   m_ntrees(n_trees),\n   m_maxdepth(max_depth),\n   m_minsplit(minsplit),\n   m_random_state(random_state)\n {\n   \n   ArrayResize(forest, n_trees);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRandomForestClassifier::~CRandomForestClassifier(void)\n {\n   for (uint i=0; i<m_ntrees; i++) //Delete the forest | all trees\n     if (CheckPointer(forest[i]) != POINTER_INVALID)\n      delete(forest[i]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRandomForestClassifier::fit(matrix &x, vector &y, bool replace=true, errors_classifier err=ERR_ACCURACY)\n {\n  matrix x_subset;\n  vector y_subset;\n  matrix data = MatrixExtend::concatenate(x, y, 1);\n  matrix temp_data = data;\n  vector preds;\n  \n  datetime time_start = GetTickCount(), current_time;\n  \n  Print(\"[ Classifier Random Forest Building ]\");\n    \n   for (uint i=0; i<m_ntrees; i++) //Build a given x number of trees\n     {\n       time_start = GetTickCount();\n       \n       temp_data = data;\n       MatrixExtend::Randomize(temp_data, m_random_state, replace); //Get randomized subsets\n       \n       if (!MatrixExtend::XandYSplitMatrices(temp_data, x_subset, y_subset)) //split the random subset into x and y subsets\n         {\n            ArrayRemove(forest,i,1); //Delete the invalid tree in a forest\n            printf(\"%s %d Failed to split data for a tree \",__FUNCTION__,__LINE__);\n            continue;\n         } \n       \n       forest[i] = new CDecisionTreeClassifier(this.m_minsplit, this.m_maxdepth); //Add the tree to the forest\n                     \n       forest[i].fit(x_subset, y_subset); //Add the trained tree to the forest\n       preds = forest[i].predict_bin(x_subset);\n       \n       current_time = GetTickCount();\n       \n       printf(\"   ==> Tree <%d> Rand Seed <%s> Accuracy Score: %.3f Time taken: %s\",i+1,m_random_state==-1?\"None\":string(m_random_state),this.err_metric(err, y_subset, preds), ConvertTime((current_time - time_start) / 1000.0));\n     }\n     \n   m_ntrees = ArraySize(forest); //The successfully build trees\n   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CRandomForestClassifier::predict(vector &x)\n {\n   vector predictions(m_ntrees); //predictions from all the trees\n    \n    for (uint i=0; i<this.m_ntrees; i++) //all trees make the predictions\n      predictions[i] = forest[i].predict_bin(x);\n      \n   vector uniques = MatrixExtend::Unique(predictions);   \n   \n   return uniques[MatrixExtend::Unique_count(predictions).ArgMax()]; //select the majority decision\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CRandomForestClassifier::predict(matrix &x)\n {\n   vector preds(x.Rows());\n   \n   for (ulong i=0; i<x.Rows(); i++)\n     preds[i] = this.predict(x.Row(i));\n  \n  return preds;     \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nstring CRandomForestClassifier::ConvertTime(double seconds)\n{\n    string time_str = \"\";\n    uint minutes = 0, hours = 0;\n\n    if (seconds >= 60)\n    {\n        minutes = (uint)(seconds / 60.0) ;\n        seconds = fmod(seconds, 1.0) * 60;\n        time_str = StringFormat(\"%d Minutes and %.3f Seconds\", minutes, seconds);\n    }\n    \n    if (minutes >= 60)\n    {\n        hours = (uint)(minutes / 60.0);\n        minutes = minutes % 60;\n        time_str = StringFormat(\"%d Hours and %d Minutes\", hours, minutes);\n    }\n\n    if (time_str == \"\")\n    {\n        time_str = StringFormat(\"%.3f Seconds\", seconds);\n    }\n\n    return time_str;\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CRandomForestClassifier::err_metric(errors_classifier err, vector &actual, vector &preds)\n {\n   return Metrics::accuracy_score(actual, preds);\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|      Random Forest for regression problems                       |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nenum errors_regressor\n  {\n   ERR_R2_SCORE,\n   ERR_ADJUSTED_R\n  };  \n  \nclass CRandomForestRegressor\n  {\nprivate:\n   uint  m_ntrees;\n   uint  m_maxdepth;\n   uint  m_minsplit;\n   int   m_random_state;\n   \n   CDecisionTreeRegressor *forest[];\n   \n   string ConvertTime(double seconds);\n   double err_metric(errors_regressor err,vector &actual,vector &preds);\n   \npublic:\n                     CRandomForestRegressor(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1);\n                    ~CRandomForestRegressor(void);\n                    \n                    void fit(matrix &x, vector &y, bool replace=true, errors_regressor err=ERR_R2_SCORE);\n                    double predict(vector &x);\n                    vector predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRandomForestRegressor::CRandomForestRegressor(uint n_trees=100, uint minsplit=NULL, uint max_depth=NULL, int random_state=-1):\n   m_ntrees(n_trees),\n   m_maxdepth(max_depth),\n   m_minsplit(minsplit),\n   m_random_state(random_state)\n {\n   \n   ArrayResize(forest, n_trees);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCRandomForestRegressor::~CRandomForestRegressor(void)\n {\n   for (uint i=0; i<m_ntrees; i++) //Delete the forest | all trees\n     if (CheckPointer(forest[i]) != POINTER_INVALID)\n      delete(forest[i]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nstring CRandomForestRegressor::ConvertTime(double seconds)\n{\n    string time_str = \"\";\n    uint minutes = 0, hours = 0;\n\n    if (seconds >= 60)\n    {\n        minutes = (uint)(seconds / 60.0) ;\n        seconds = fmod(seconds, 1.0) * 60;\n        time_str = StringFormat(\"%d Minutes and %.3f Seconds\", minutes, seconds);\n    }\n    \n    if (minutes >= 60)\n    {\n        hours = (uint)(minutes / 60.0);\n        minutes = minutes % 60;\n        time_str = StringFormat(\"%d Hours and %d Minutes\", hours, minutes);\n    }\n\n    if (time_str == \"\")\n    {\n        time_str = StringFormat(\"%.3f Seconds\", seconds);\n    }\n\n    return time_str;\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CRandomForestRegressor::fit(matrix &x, vector &y, bool replace=true, errors_regressor err=ERR_R2_SCORE)\n {\n  matrix x_subset;\n  vector y_subset;\n  matrix data = MatrixExtend::concatenate(x, y, 1);\n  matrix temp_data = data;\n  \n  vector preds;\n  datetime time_start = GetTickCount(), current_time;\n  \n  Print(\"[ Regressor Random Forest Building ]\");\n    \n   for (uint i=0; i<m_ntrees; i++)\n     {\n       time_start = GetTickCount();\n       \n       temp_data = data;\n       MatrixExtend::Randomize(temp_data, m_random_state, replace);\n       \n       if (!MatrixExtend::XandYSplitMatrices(temp_data, x_subset, y_subset)) //Get randomized subsets\n         {  \n            ArrayRemove(forest,i,1); //Delete the invalid tree in a forest\n            printf(\"%s %d Failed to split data for a tree \",__FUNCTION__,__LINE__);\n            continue;\n         }\n       \n       forest[i] = new CDecisionTreeRegressor(this.m_minsplit, this.m_maxdepth);       \n       forest[i].fit(x_subset, y_subset); //Add the trained tree to the forest\n       preds = forest[i].predict(x_subset);\n       \n       current_time = GetTickCount();\n       \n       printf(\"   ==> Tree <%d> Rand Seed <%s> R_2 Score: %.3f Time taken: %s\",i+1,m_random_state==-1?\"None\":string(m_random_state),this.err_metric(err, y_subset, preds), ConvertTime((current_time - time_start) / 1000.0));\n     }\n     \n   m_ntrees = ArraySize(forest); //The successfully build trees  \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CRandomForestRegressor::predict(vector &x)\n {\n    vector predictions(m_ntrees); //predictions from all the trees\n    \n    for (uint i=0; i<this.m_ntrees; i++)\n      predictions[i] = forest[i].predict(x);\n\n   return predictions.Mean();   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CRandomForestRegressor::predict(matrix &x)\n {\n   vector preds(x.Rows());\n   \n   for (ulong i=0; i<x.Rows(); i++)\n     preds[i] = this.predict(x.Row(i));\n  \n  return preds;     \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CRandomForestRegressor::err_metric(errors_regressor err,vector &actual,vector &preds)\n {\n   double acc = 0;\n   switch(err)\n     {\n      case ERR_R2_SCORE:\n        acc = Metrics::r_squared(actual, preds);\n        break;\n      case ERR_ADJUSTED_R:\n        acc = Metrics::adjusted_r(actual, preds);\n        break; \n     }\n     \n   return acc;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Linear Models/Logistic Regression.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                 MatrixExtend::mqh |\n//|                                  Copyright 2022, Omega Joctan  . |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, MetaQuotes Ltd.\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include <MALE5\\Utils.mqh>\n#include <MALE5\\Sklearn\\metrics.mqh>\n#include <MALE5\\Numpy\\Numpy.mqh>\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass CLogisticRegression\n  {\nprivate:\n   CNumpy            np;\n   vector            classes_in_data;\n\n   bool              istrained;\n   bool              checkIsTrained(string func)\n     {\n      if(!istrained)\n        {\n         Print(func,\" Tree not trained, Call fit function first to train the model\");\n         return false;\n        }\n      return (true);\n     }\n\n   bool              CheckSamplesSize(string func, ulong size)\n     {\n      if(size != m_features)\n        {\n         printf(\"%s x sample size doesn't align with the training data m_features %d\",func, size);\n         return false;\n        }\n      return true;\n     }\n\n   matrix            weights;\n   double            bias;\n\n   //---\n\n   uint              m_epochs;\n   double            m_alpha;\n   double            m_tol;\n   ulong             m_features;\n   int               m_random_seed;\n\npublic:\n                     CLogisticRegression(uint epochs=10, double alpha=0.01, double tol=1e-8, int random_seed = 0);\n                    ~CLogisticRegression(void);\n\n\n   void              fit(matrix &x, vector &y);\n\n   int               predict(vector &x);\n   vector            predict(matrix &x);\n   double            predict_proba(vector &x);\n   vector            predict_proba(matrix &x);\n\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCLogisticRegression::CLogisticRegression(uint epochs=10, double alpha=0.01, double tol=1e-8, int random_seed=0)\n   :istrained(false),\n    m_epochs(epochs),\n    m_alpha(alpha),\n    m_tol(tol),\n    m_random_seed(random_seed)\n  {\n\n  }\n//+------------------------------------------------------------------+\n//| This is where the logistic model gets trained                    |\n//+------------------------------------------------------------------+\nvoid CLogisticRegression::fit(matrix &x, vector &y)\n  {\n   ulong m = x.Rows(), n = x.Cols();\n   m_features = n;\n   \n   np.random.seed(m_random_seed);\n   vector rand_v = np.random.uniform(-1, 1, (uint)n);\n   this.weights = np.expand_dims(rand_v, 1);\n   \n   //---\n   \n   matrix dw; //derivative wrt weights &\n   double db; //bias respectively\n   vector preds;\n\n   istrained = true;\n\n   double prev_cost = -DBL_MAX, cost =0;\n   for(ulong i=0; i<m_epochs; i++)\n     {\n      preds = this.predict_proba(x);\n\n      //-- Computing gradient(s)\n\n      matrix error = np.expand_dims(preds - y,  1);\n\n      dw = (1/(double)m) * x.Transpose().MatMul(error);\n      db = (1/(double)m) * (preds - y).Sum();\n\n      cost = Metrics::mse(y, preds);\n\n      printf(\"---> Logistic regression build epoch [%d/%d] mse %.5f\",i+1,m_epochs, cost);\n\n      this.weights -= this.m_alpha * dw;\n      this.bias -= this.bias * db;\n\n      if(MathAbs(prev_cost - cost) < this.m_tol)\n        {\n         Print(\"Converged!!!\");\n         break;\n        }\n\n      prev_cost = cost;\n     }\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCLogisticRegression::~CLogisticRegression(void)\n  {\n\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint CLogisticRegression::predict(vector &x)\n  {\n   if(!checkIsTrained(__FUNCTION__))\n      return 0;\n\n   if(!CheckSamplesSize(__FUNCTION__,x.Size()))\n      return 0;\n\n   matrix x_mat = np.expand_dims(x, 1);\n   matrix preds = (x_mat.MatMul(this.weights) + this.bias);\n\n   preds.Activation(preds, AF_HARD_SIGMOID);\n\n   if(preds.Rows()>1)\n     {\n      printf(\"%s The outcome from a sigmoid must be a scalar value\",__FUNCTION__);\n      return 0;\n     }\n   return (int)(preds[0][0]>=0.5);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CLogisticRegression::predict(matrix &x)\n  {\n   vector v(x.Rows());\n   for(ulong i=0; i<x.Rows(); i++)\n      v[i] = this.predict(x.Row(i));\n\n   return v;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CLogisticRegression::predict_proba(vector &x)\n  {\n   if(!checkIsTrained(__FUNCTION__))\n      return 0;\n\n   matrix x_mat = np.expand_dims(x, 1);\n   matrix preds = (x_mat.MatMul(this.weights) + this.bias);\n\n   preds.Activation(preds, AF_HARD_SIGMOID);\n\n   if(preds.Rows()>1)\n     {\n      printf(\"%s The outcome from a sigmoid must be a scalar value\",__FUNCTION__);\n      return 0;\n     }\n   return preds[0][0];\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CLogisticRegression::predict_proba(matrix &x)\n  {\n   vector v(x.Rows());\n   for(ulong i=0; i<x.Rows(); i++)\n      v[i] = this.predict_proba(x.Row(i));\n\n   return v;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/Linear Models/README.md",
    "content": "## Theory Overview: Linear Regression\n\nLinear regression is a statistical method for modeling the relationship between a dependent variable (what you want to predict) and one or more independent variables (what you are using to predict). It assumes a linear relationship between the variables, meaning the change in the dependent variable is proportional to the change in the independent variable(s).\n\nThe goal of linear regression is to find a line (for single independent variable) or hyperplane (for multiple independent variables) that best fits the data points. This line/hyperplane is represented by the following equation:\n\n`y = β₀ + β₁x₁ + β₂x₂ + ... + βₙxₙ`\n\nwhere:\n\n* `y` is the dependent variable\n* `x₁`, `x₂`, ..., `xₙ` are the independent variables\n* `β₀` is the intercept (the y-value where the line/hyperplane crosses the y-axis)\n* `β₁`, `β₂`, ..., `βₙ` are the coefficients (slopes) of the independent variables\n\nBy fitting the line/hyperplane to the data, linear regression allows us to:\n\n* **Make predictions:** We can use the fitted model to predict the dependent variable for new data points based on their independent variable values.\n* **Understand relationships:** The coefficients represent the strength and direction of the relationships between the independent and dependent variables.\n\n## CLinearRegression Class Documentation\n\nThe `CLinearRegression` class implements linear regression functionality. It provides methods for both training and prediction:\n\n**Public Functions:**\n\n* `CLinearRegression(void)` Default constructor.\n* `~CLinearRegression(void)` Destructor.\n* `void fit_LeastSquare(matrix &x, vector &y)` Fits the model using the least squares method. \n    * This method trains the model by finding the coefficients (Betas) that minimize the sum of squared errors between the predicted and actual values.\n    * Requires `x` (matrix of independent variables) and `y` (vector of dependent variables) as input.\n* `void fit_GradDescent(matrix &x, vector &y, double alpha, uint epochs = 1000)` Fits the model using gradient descent.\n    * This method trains the model iteratively, updating the coefficients based on the calculated gradients (slopes of the error function).\n    * Requires `x` (matrix of independent variables), `y` (vector of dependent variables), `alpha` (learning rate), and optional `epochs` (number of iterations) as input.\n* `double predict(vector &x)` Predicts the dependent variable for a new data point represented by the input vector `x`.\n    * Requires a vector containing the values for the independent variables of the new data point.\n    * Assumes the model is already trained (trained flag checked internally).\n* `vector predict(matrix &x)` Predicts the dependent variables for multiple new data points represented by the input matrix `x`.\n    * Requires a matrix where each row represents a new data point with its independent variable values.\n    * Assumes the model is already trained (trained flag checked internally).\n\n**Additional Notes:**\n\n* The class uses internal member variables to store the trained coefficients (`Betas` and `Betas_v`).\n* Internal helper functions (`checkIsTrained`, `TrimNumber`, `dx_wrt_bo`, `dx_wrt_b1`) are not directly accessible from the user but support the core functionalities.\n* The class checks if the model is trained before allowing predictions using the `checkIsTrained` function.\n\n\n\n## CLogisticRegression Class: Logistic Regression\n\nThe `CLogisticRegression` class provides functionalities for implementing logistic regression in MQL5. This statistical method allows you to model the probability of a binary outcome (belonging to one of two classes) based on independent variables.\n\n**Public Functions:**\n\n* `CLogisticRegression(uint epochs=10, double alpha=0.01, double tol=1e-8)` Constructor, allows setting hyperparameters (epochs, learning rate, tolerance) for training.\n* `~CLogisticRegression(void)` Destructor.\n* `void fit(matrix &x, vector &y)` Trains the model using the provided training data (`x` - independent variables, `y` - binary classification labels).\n    * Internally performs gradient descent optimization to find the optimal weights and bias for the model.\n* `int predict(vector &x)` Predicts the class label (0 or 1) for a new data point represented by the input vector `x`.\n    * Assumes the model is already trained (checked internally).\n* `vector predict(matrix &x)` Predicts class labels for multiple new data points represented by the input matrix `x`.\n    * Each row in the matrix represents a new data point.\n    * Assumes the model is already trained (checked internally).\n* `double predict_proba(vector &x)` Predicts the probability of belonging to class 1 for a new data point represented by the input vector `x`.\n    * Assumes the model is already trained (checked internally).\n* `vector predict_proba(matrix &x)` Predicts the probabilities of belonging to class 1 for multiple new data points represented by the input matrix `x`.\n    * Each row in the matrix represents a new data point.\n    * Assumes the model is already trained (checked internally).\n\n**Mathematical Theory (Basic Overview):**\n\nLogistic regression uses the sigmoid function to map the linear combination of the input features (weighted sum) to a probability between 0 and 1. \n\nThe mathematical formula for the logistic function is:\n\n```\nf(z) = 1 / (1 + exp(-z))\n```\n\nwhere:\n\n* `z` is the linear combination of weights and features (z = w₁x₁ + w₂x₂ + ... + wₙxₙ + b)\n* `w_i` are the weights for each feature\n* `b` is the bias term\n* `f(z)` is the predicted probability of belonging to class 1\n\nBy adjusting the weights and bias through training (minimizing the error between predicted and actual labels), the logistic regression model learns to distinguish between the two classes based on the input features and predict probabilities accordingly.\n\n**Additional Notes:**\n\n* The class utilizes gradient descent to optimize the weights and bias during training.\n* Hyperparameter tuning (epochs, learning rate, tolerance) can significantly impact model performance and should be considered based on the specific data and task.\n\n\n\n## CPolynomialRegression Class: Polynomial Regression\n\nThe `CPolynomialRegression` class provides functionalities for implementing polynomial regression in MQL5. This technique extends linear regression by fitting a higher-degree polynomial function to the data, allowing for more complex relationships between the independent and dependent variables.\n\n**Public Functions:**\n\n* `CPolynomialRegression(int degree=2)` Constructor, allows setting the degree of the polynomial (default is 2).\n* `~CPolynomialRegression(void)` Destructor.\n* `void BIC(ulong k, vector &bic, int &best_degree)` Calculates the Bayesian Information Criterion (BIC) for different polynomial degrees (`k`) and recommends the \"best\" degree based on the lowest BIC value (stored in `best_degree`).\n    * Requires `k` (vector of degree values to evaluate), `bic` (output vector to store BIC values), and `best_degree` (output variable to store the recommended degree).\n* `void fit(matrix &x, vector &y)` Trains the model using the provided training data (`x` - independent variables, `y` - dependent variables).\n    * Internally fits the polynomial function to the data and stores the coefficients in the `Betas` and `Betas_v` member variables.\n* `double predict(vector &x)` Predicts the dependent variable for a new data point represented by the input vector `x`.\n    * Assumes the model is already trained (trained flag not explicitly mentioned but potentially implemented internally).\n* `vector predict(matrix &x)` Predicts the dependent variables for multiple new data points represented by the input matrix `x`.\n    * Each row in the matrix represents a new data point.\n    * Assumes the model is already trained (trained flag not explicitly mentioned but potentially implemented internally).\n\n**Mathematical Theory (Basic Overview):**\n\nPolynomial regression models the relationship between the independent and dependent variables using a polynomial function of the form:\n\n```\ny = β₀ + β₁x + β₂x² + ... + βₙxⁿ\n```\n\nwhere:\n\n* `y` is the dependent variable\n* `x` is the independent variable\n* `β₀`, `β₁`, ..., `βₙ` are the coefficients of the polynomial (model parameters)\n* `n` is the degree of the polynomial (set during object creation or through the `BIC` function)\n\nBy increasing the degree of the polynomial, the model can capture more complex non-linear relationships in the data. However, it is crucial to find a balance between model complexity and overfitting (memorizing the training data poorly generalizing to unseen data).\n\n**Additional Notes:**\n\n* The `BIC` function can be used to help select an appropriate polynomial degree by balancing model complexity and goodness-of-fit.\n* Choosing a high polynomial degree without sufficient data can lead to overfitting, so careful consideration and potentially additional techniques like regularization might be necessary.\n\n\n**References**\n* [Data Science and Machine Learning (Part 01): Linear Regression](https://www.mql5.com/en/articles/10459)\n* [Data Science and Machine Learning (Part 02): Logistic Regression](https://www.mql5.com/en/articles/10626)\n* [Data Science and Machine Learning (Part 07): Polynomial Regression](https://www.mql5.com/en/articles/11477)\n* [Data Science and Machine Learning (Part 10): Ridge Regression ](https://www.mql5.com/en/articles/11735)\n"
  },
  {
    "path": "Sklearn/Linear Models/Ridge.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                             Ridge Regression.mqh |\n//|                                  Copyright 2022, MetaQuotes Ltd. |\n//|                                             https://www.mql5.com |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n \n#include <MALE5\\Sklearn\\preprocessing.mqh>\n#include <MALE5\\Utils.mqh>\n#include \"Linear Regression.mqh\"\n\n//+------------------------------------------------------------------+\n\nclass CRidgeregression\n  {\n  \n   protected: \n                        matrix XMatrix; //matrix of independent variables\n                        matrix YMatrix;\n                        vector yVector; // Vector of target variables\n                        matrix Id_matrix; //Identity matrix\n                        \n                        matrix Betas;\n                        ulong  n; //No of samples\n                        ulong  k; //No of regressors \n                        \n   public:\n                        CRidgeregression(matrix &_matrix);\n                       ~CRidgeregression(void);\n                         \n                       double RSS;\n                       double Lr_accuracy;\n                       \n                       vector L2Norm(double lambda); //Ridge regression\n                        \n  };\n  \n//+------------------------------------------------------------------+\n/*\nCRidgeregression::CRidgeregression(matrix &_matrix)\n {\n    n = _matrix.Rows();\n    k = _matrix.Cols();\n    \n    MatrixExtend::XandYSplitMatrices(_matrix,XMatrix,yVector);\n    \n    YMatrix = MatrixExtend::VectorToMatrix(yVector);\n    \n//---\n\n    Id_matrix.Resize(k,k);\n    \n    Id_matrix.Identity();\n\n }\n*/\n//+------------------------------------------------------------------+\n\nCRidgeregression::~CRidgeregression(void)\n {\n   ZeroMemory(XMatrix);\n   ZeroMemory(yVector);\n   ZeroMemory(yVector);\n   ZeroMemory(Id_matrix); \n }\n \n//+------------------------------------------------------------------+\n/*\nvector CRidgeregression::L2Norm(double lambda)\n {    \n   matrix design = MatrixExtend::DesignMatrix(XMatrix);\n   \n   matrix XT = design.Transpose();\n   \n   matrix XTX = XT.MatMul(design);\n   \n   matrix lamdaxI = lambda * Id_matrix;\n   \n   matrix sum_matrix = XTX + lamdaxI;\n   \n   matrix Inverse_sum = sum_matrix.Inv();\n   \n   matrix XTy = XT.MatMul(YMatrix);\n   \n   Betas = Inverse_sum.MatMul(XTy);\n \n   #ifdef DEBUG_MODE\n      //Print(\"Betas\\n\",Betas);\n   #endif \n   \n  return(MatrixExtend::MatrixToVector(Betas));\n }\n//+------------------------------------------------------------------+\n*/"
  },
  {
    "path": "Sklearn/Naive Bayes/Naive Bayes.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                  Naive Bayes.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\Utils.mqh> \n//+------------------------------------------------------------------+\n//|              N  A  I  V  E     B  A  Y  E S                      |\n//|                                                                  |\n//|   suitable for classification of discrete values, that have      |\n//|   been load to a matrix using the method ReadCSVEncode from      |\n//|   CUtils::mqh                                              |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CNaiveBayes\n  {\nprotected:\n                     uint n_features;\n                     vector y_target;\n                     \n                     vector class_proba; //prior class probability\n                     vector features_proba; //features probability\n                       \n                     vector c_prior_proba; //class prior probability\n                     vector c_evidence;    //class evidence\n                     \n                     vector calcProba(vector &v_features);\n                     \npublic:\n                     vector classes;       //classes available \n                     \n                     CNaiveBayes(void);\n                    ~CNaiveBayes(void);\n                    \n                     void fit(matrix &x, vector &y);\n                     int predict(vector &x);\n                     vector predict(matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCNaiveBayes::CNaiveBayes(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CNaiveBayes::fit(matrix &x, vector &y)\n {\n  ulong samples = x.Rows(),\n        features = x.Cols();\n  \n  vector unique = CUtils::Unique_count(y);\n  \n  this.class_proba = unique / samples;\n  \n  if (MQLInfoInteger(MQL_DEBUG))\n    Print(\"class probabilities: \",class_proba);\n  \n    \n \n/*\n   y_target = y;\n   n_features = x.Cols();\n   \n   classes = CUtils::Unique(y);\n   \n   c_evidence.Resize((ulong)classes.Size());\n   \n   n = y.Size();\n   \n   if (n==0) { Print(\"--> n == 0 | Naive Bayes class failed\"); return; }\n   \n//---\n\n   vector v = {};\n   for (ulong i=0; i<c_evidence.Size(); i++)\n       {\n         v = CUtils::Search(y,classes[i]);\n         \n         c_evidence[i] = (int)v.Size();\n       }\n\n//---    \n   \n   c_prior_proba.Resize(classes.Size());\n   \n   for (ulong i=0; i<classes.Size(); i++)\n      c_prior_proba[i] = c_evidence[i]/(double)n;\n   \n  \n   Print(\"---> GROUPS \",classes);\n   Print(\"Prior Class Proba \",c_prior_proba,\"\\nEvidence \",c_evidence);\n*/  \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCNaiveBayes::~CNaiveBayes(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint CNaiveBayes::predict(vector &x)\n {   \n   vector v = calcProba(x);\n   \n   double sum = v.Sum();\n   \n   for (ulong i=0; i<v.Size(); i++) //converting the values into probabilities\n      v[i] = NormalizeDouble(v[i]/sum,2);       \n   \n   vector p = v;\n   \n   #ifdef   DEBUG_MODE\n      Print(\"Probabilities \",p);\n   #endif \n   \n   return((int)classes[p.ArgMax()]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CNaiveBayes::predict(matrix &x)\n {\n  ulong rows = x.Rows();\n \n  vector v(rows), pred(rows); \n  \n   for (ulong i=0; i<rows; i++)\n    { \n       v = x.Row(i);\n       pred[i] = predict(v);\n    }\n    \n   return pred;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n/*\nvector CNaiveBayes::calcProba(vector &v_features)\n {\n    vector proba_v(classes.Size()); //vector to return\n    \n    if (v_features.Size() != n_features)\n      {\n         printf(\"FATAL | Can't calculate probability,  fetures columns size = %d is not equal to x_matrix columns =%d\",v_features.Size(),n_features);\n         return proba_v;\n      }\n\n//---\n    \n    vector v = {}; \n    \n    for (ulong c=0; c<classes.Size(); c++)\n      {\n        double proba = 1;\n          for (ulong i=0; i<n_features; i++)\n            {\n                v = x_matrix.Col(i);\n                \n                int count =0;\n                for (ulong j=0; j<v.Size(); j++)\n                  {\n                     if (v_features[i] == v[j] && classes[c] == y[j])\n                        count++;\n                  }\n                  \n                proba *= count==0 ? 1 : count/(double)c_evidence[c]; //do not calculate if there isn't enough evidence'\n            }\n          \n        proba_v[c] = proba*c_prior_proba[c];\n     }\n     \n    return proba_v;\n }*/\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//|            NORMAL DISTRIBUTION CLASS                             |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CNormDistribution\n  {\n\npublic:\n   \n   double m_mean; //Assign the value of the mean\n   double m_std;  //Assign the value of Variance\n   \n                     CNormDistribution(void);\n                    ~CNormDistribution(void);\n                    \n                     double PDF(double x); //Probability density function\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nCNormDistribution::CNormDistribution(void)\n {\n   \n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nCNormDistribution::~CNormDistribution(void)\n {\n   ZeroMemory(m_mean);\n   ZeroMemory(m_std);\n }\n \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\ndouble CNormDistribution::PDF(double x)\n {\n   double nurm = MathPow((x - m_mean),2)/(2*MathPow(m_std,2));\n   nurm = exp(-nurm);\n   \n   double denorm = 1.0/(MathSqrt(2*M_PI*MathPow(m_std,2)));\n      \n  return(nurm*denorm);\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|          GAUSSIAN NAIVE BAYES CLASS                              |\n//|                                                                  |\n//|   Suitable for classification based on features with             |\n//|   continuous variables,                                          |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\n/*\nclass CGaussianNaiveBayes\n  {\n   protected:\n   \n      CNormDistribution norm_distribution;\n\n      vector            c_prior_proba; //prior probability\n      vector            c_evidence;\n      ulong             n;\n      \n      ulong              m_cols;  //columns in x_matrix\n      vector             calcProba(vector &v_features);\n   \n   public:              \n   \n      vector            classes; //Target classes     \n             \n                        CGaussianNaiveBayes(void);\n                       ~CGaussianNaiveBayes(void);\n                        \n                        void fit(matrix &x, vector &y);\n                        \n                        int predict_bin(vector &x);\n                        vector predict_bin(matrix &x);\n                        vector predict_proba(vector &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCGaussianNaiveBayes::CGaussianNaiveBayes(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CGaussianNaiveBayes::fit(matrix &x, vector &y)\n { \n   \n   classes = CUtils::Unique(y);\n   m_cols = n_features;\n    \n//---\n   \n   c_evidence.Resize((ulong)classes.Size());\n   \n   n = y.Size();\n   \n   if (n==0) { Print(\"---> n == 0 | Gaussian Naive Bayes class failed\"); return; }\n   \n//---\n   \n   vector v = {};\n   for (ulong i=0; i<c_evidence.Size(); i++)\n       {          \n         v = CUtils::Search(y, classes[i]);\n         \n         c_evidence[i] = (int)v.Size();\n       }\n   \n   c_prior_proba.Resize(classes.Size());\n   \n   for (ulong i=0; i<classes.Size(); i++)\n      c_prior_proba[i] = c_evidence[i]/(double)n;\n\n//---       \n   \n   Print(\"---> GROUPS \",classes);\n   Print(\"\\n---> Prior_proba \",c_prior_proba,\" Evidence \",c_evidence);\n\n//---\n\n   during_training = false; \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCGaussianNaiveBayes::~CGaussianNaiveBayes(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nint CGaussianNaiveBayes::predict_bin(vector &x)\n {     \n   if (x.Size() != m_cols)\n     {\n       Print(\"CRITICAL | The given x have different size than the trained x\");\n       return (-1);\n     }\n   \n   vector p = calcProba(x);\n   \n   return((int)classes[p.ArgMax()]);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CGaussianNaiveBayes::predict_proba(vector &x)\n {\n  vector x = x;\n  vector ret_v = {};\n  \n   if (x.Size() != m_cols)\n     {\n       Print(\"CRITICAL | The given x have different size than the trained x\");\n       return (ret_v);\n     }\n        \n   ret_v = calcProba(x);\n   \n   return (ret_v);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CGaussianNaiveBayes::predict_bin(matrix &x)\n {\n  ulong rows = x.Rows();\n  vector v(rows), pred(rows); \n  \n   for (ulong i=0; i<rows; i++)\n    { \n       v = x.Row(i);\n       \n       pred[i] = predict_bin(v);\n    }\n   \n   return pred;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nvector CGaussianNaiveBayes::calcProba(vector &v_features)\n {    \n    vector proba_v(classes.Size()); //vector to return\n    proba_v.Fill(-1);\n    \n    if (v_features.Size() != m_cols)\n      {\n         printf(\"FATAL | Can't calculate probability, fetures columns size = %d is not equal to x_matrix columns =%d\",v_features.Size(),m_cols);\n         return proba_v;\n      }\n\n//---\n    \n    vector v = {}; \n    \n    for (ulong c=0; c<classes.Size(); c++)\n      {\n        double proba = 1;\n          for (ulong i=0; i<m_cols; i++)\n            {\n                v = x_matrix.Col(i);\n                \n                int count =0;\n                vector calc_v = {};\n                \n                for (ulong j=0; j<v.Size(); j++)\n                  {\n                     if (classes[c] == y[j])\n                       {\n                         count++;\n                         calc_v.Resize(count);\n                         \n                         calc_v[count-1] = v[j];\n                       }\n                  } \n                \n                norm_distribution.m_mean = calc_v.Mean(); //Assign these to Gaussian Normal distribution\n                norm_distribution.m_std = calc_v.Std();   \n                 \n                \n                #ifdef DEBUG_MODE\n                  printf(\"mean %.5f std %.5f \",norm_distribution.m_mean,norm_distribution.m_std);\n                #endif \n                \n                proba *= count==0 ? 1 : norm_distribution.PDF(v_features[i]); //do not calculate if there isn't enought evidence'\n            }\n          \n        proba_v[c] = proba*c_prior_proba[c]; //Turning the probability density into probability\n        \n        #ifdef DEBUG_MODE\n         Print(\">> Proba \",proba,\" prior proba \",c_prior_proba);\n        #endif \n     }\n\n//--- Normalize probabilities\n    \n    proba_v = proba_v / proba_v.Sum();\n    \n    return proba_v;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n*/"
  },
  {
    "path": "Sklearn/Naive Bayes/README.md",
    "content": "## Naive Bayes Classifier\n\nThis documentation explains the `CNaiveBayes` class in MQL5, which implements a **Naive Bayes classifier** for classification tasks.\n\n**I. Naive Bayes Theory:**\n\nNaive Bayes is a probabilistic classifier based on **Bayes' theorem**. It assumes that the features used for classification are **independent** of each other given the class label. This simplifies the calculations involved in making predictions.\n\n**II. CNaiveBayes Class:**\n\nThe `CNaiveBayes` class provides functionalities for training and using a Naive Bayes classifier in MQL5:\n\n**Public Functions:**\n\n* **CNaiveBayes(void):** Constructor.\n* **~CNaiveBayes(void):** Destructor.\n* **void fit(matrix &x, vector &y):** Trains the model on the provided data (`x` - features, `y` - target labels).\n* **int predict(vector &x):** Predicts the class label for a single input vector.\n* **vector predict(matrix &x):** Predicts class labels for all rows in the input matrix.\n\n**Internal Variables:**\n\n* `n_features`: Number of features in the data.\n* `y_target`: Vector of target labels used during training.\n* `classes`: Vector containing the available class labels.\n* `class_proba`: Vector storing the prior probability of each class.\n* `features_proba`: Matrix storing the conditional probability of each feature value given each class.\n* `c_prior_proba`: Vector storing the calculated prior probability of each class after training.\n* `c_evidence`: Vector storing the calculated class evidence for a new data point.\n* `calcProba(vector &v_features)`: Internal function (not directly accessible) that likely calculates the class probabilities for a given feature vector.\n\n**III. Class Functionality:**\n\n1. **Training:**\n    * The `fit` function takes the input data (features and labels) and performs the following:\n        * Calculates the prior probability of each class (number of samples belonging to each class divided by the total number of samples).\n        * Estimates the conditional probability of each feature value given each class (using techniques like Laplace smoothing to handle unseen features).\n    * These probabilities are stored in the internal variables for later use in prediction.\n\n2. **Prediction:**\n    * The `predict` functions take a new data point (feature vector) and:\n        * Calculate the class evidence for each class using Bayes' theorem, considering the prior probabilities and conditional probabilities of the features.\n        * The class with the **highest class evidence** is predicted as the most likely class for the new data point.\n\n**IV. Additional Notes:**\n\n* The class assumes the data is already preprocessed and ready for use.\n\n**Reference**\n* [Data Science and Machine Learning (Part 11): Naïve Bayes, Probability theory in Trading](https://www.mql5.com/en/articles/12184)"
  },
  {
    "path": "Sklearn/Naive Bayes/naive bayes visuals.py",
    "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Tue Feb 14 10:37:36 2023\n\n@author: Omega Joctan\n\"\"\"\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Change this directory to be the one containing the Files\n\ndirectory = r\"C:\\Users\\Omega Joctan\\AppData\\Roaming\\MetaQuotes\\Terminal\\F4F6C6D7A7155578A6DEA66D12B1D40D\\MQL5\\Files\\NAIVE BAYES\"\n\ndata = pd.read_csv(f\"{directory}\\\\vars.csv\")\n\nfor var in data:\n    sns.distplot(data[var])\n    plt.show()"
  },
  {
    "path": "Sklearn/Neighbors/KNN_nearest_neighbors.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                        KNN_nearest_neighbors.mqh |\n//|                                    Copyright 2022, Omega Joctan. |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Omega Joctan.\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n\nbool isdebug = true;\n\n//+------------------------------------------------------------------+ \n\nclass CKNNNearestNeighbors\n  {\nprivate:\n   uint              k;\n   matrix<double>    Matrix;\n   ulong             m_rows, m_cols;\n   vector            m_target;\n   vector            m_classesVector;\n   matrix            m_classesMatrix;\n\n   double            Euclidean_distance(const vector &v1, const vector &v2);\n   vector            ClassVector(); //global vector of target classes\n   void              MatrixRemoveRow(matrix &mat, ulong row);\n   void              VectorRemoveIndex(vector &v, ulong index);\n   double            Mse(vector &A, vector &P);\npublic:\n                     CKNNNearestNeighbors(matrix<double> &Matrix_, uint k_);\n                     CKNNNearestNeighbors(matrix<double> &Matrix_);\n                    ~CKNNNearestNeighbors(void);\n\n   int               KNNAlgorithm(vector &vector_);\n   vector            CrossValidation_LOOCV(uint initial_k = 0, uint final_k=1); //Leave One out Cross Validation (LOOCV)\n   matrix            ConfusionMatrix(vector &A,vector &P);\n   float             TrainTest(double train_size = 0.7); //returns accuracy of the tested dataset\n  };\n\n//+------------------------------------------------------------------+\n\nCKNNNearestNeighbors:: CKNNNearestNeighbors(matrix<double> &Matrix_, uint k_)\n  {\n   k = k_;\n   \n   if(k %2 ==0)\n      {\n         k = k+1;\n         if (isdebug)\n            printf(\"K = %d is an even number, It will be added by One so it becomes an odd Number %d\", k_, k);\n      }\n\n   Matrix.Copy(Matrix_);\n\n   m_rows = Matrix.Rows();\n   m_cols = Matrix.Cols();\n\n   m_target = Matrix.Col(m_cols-1);\n   m_classesVector = ClassVector();\n   \n   if (isdebug)\n      Print(\"classes vector | Neighbors \", m_classesVector);\n  }\n\n//+------------------------------------------------------------------+\n\nCKNNNearestNeighbors::CKNNNearestNeighbors(matrix<double> &Matrix_)\n  {\n   Matrix.Copy(Matrix_);\n\n   k = (int)round(MathSqrt(Matrix.Rows()));\n   k = k%2 ==0 ? k+1 : k; //make sure the value of k ia an odd number\n\n   m_rows = Matrix.Rows();\n   m_cols = Matrix.Cols();\n\n   m_target = Matrix.Col(m_cols-1);\n   m_classesVector = ClassVector();\n   Print(\"classes vector | Neighbors \", m_classesVector);\n  }\n\n//+------------------------------------------------------------------+\n\nCKNNNearestNeighbors::~CKNNNearestNeighbors(void)\n  {\n   ZeroMemory(k);\n   ZeroMemory(m_classesVector);\n   ZeroMemory(m_classesMatrix);\n  }\n\n//+------------------------------------------------------------------+ \n\nint CKNNNearestNeighbors::KNNAlgorithm(vector &vector_)\n  {\n   vector vector_2 = {};\n   vector euc_dist;\n   euc_dist.Resize(m_rows);\n\n   //matrix temp_matrix = Matrix;\n   //temp_matrix.Resize(Matrix.Rows(), Matrix.Cols()-1); //remove the last column of independent variables\n   \n   for(ulong i=0; i<m_rows; i++)\n     {\n      vector_2 = Matrix.Row(i);\n      vector_2.Resize(m_cols-1);\n       \n      euc_dist[i] = NormalizeDouble(Euclidean_distance(vector_, vector_2), 5);\n     }\n\n//---   \n\n   if(isdebug)\n     {\n      matrix dbgMatrix = Matrix; //temporary debug matrix\n      dbgMatrix.Resize(dbgMatrix.Rows(), dbgMatrix.Cols()+1);\n      dbgMatrix.Col(euc_dist, dbgMatrix.Cols()-1);\n\n      //Print(\"Matrix w Euclidean Distance\\n\",dbgMatrix);\n\n      ZeroMemory(dbgMatrix);\n     }\n\n//---\n   \n   uint size = (uint)m_target.Size();\n\n   double tarArr[];\n   ArrayResize(tarArr, size);\n   double eucArray[];\n   ArrayResize(eucArray, size);\n\n   for(ulong i=0; i<size; i++)  //convert the vectors to array\n     {      \n      tarArr[i] = m_target[i];\n      eucArray[i] = euc_dist[i];\n     }\n\n   double track[], NN[];\n   ArrayCopy(track, tarArr);\n\n\n   int max; \n   \n   for(int i=0; i<(int)m_target.Size(); i++)\n     {\n      if(ArraySize(track) > (int)k)\n        {\n         max = ArrayMaximum(eucArray);\n         ArrayRemove(eucArray, max, 1);\n         ArrayRemove(track, max, 1);\n        }\n     }\n   ArrayCopy(NN, eucArray);\n/*\n   Print(\"NN \");\n   ArrayPrint(NN);\n   Print(\"Track \");\n   ArrayPrint(track);\n*/\n//--- Voting process\n\n   vector votes(m_classesVector.Size());\n\n   for(ulong i=0; i<votes.Size(); i++)\n     {\n      int count = 0;\n      for(ulong j=0; j<track.Size(); j++)\n        {\n         if(m_classesVector[i] == track[j])\n            count++;\n        }\n\n      votes[i] = (double)count;\n\n      if(votes.Sum() == k)  //all members have voted\n         break;\n     }\n\n   if(isdebug)\n      Print(vector_, \" belongs to class \", (int)m_classesVector[votes.ArgMax()]);\n      \n     return((int)m_classesVector[votes.ArgMax()]);\n  }\n\n//+------------------------------------------------------------------+\n \ndouble CKNNNearestNeighbors:: Euclidean_distance(const vector &v1, const vector &v2)\n  {\n   double dist = 0;\n\n   if(v1.Size() != v2.Size())\n      Print(__FUNCTION__, \" v1 and v2 not matching in size\");\n   else\n     {\n      double c = 0;\n      for(ulong i=0; i<v1.Size(); i++)\n         c += MathPow(v1[i] - v2[i], 2);\n\n      dist = MathSqrt(c);\n     }\n\n   return(dist);\n  }\n\n//+------------------------------------------------------------------+\n \nvector CKNNNearestNeighbors::ClassVector()\n  {\n   vector t_vectors = Matrix.Col(m_cols-1); //target variables are found on the last column in the matrix\n   vector temp_t = t_vectors, v = {t_vectors[0]};\n\n   for(ulong i=0, count =1; i<m_rows; i++)  //counting the different neighbors\n     {\n      for(ulong j=0; j<m_rows; j++)\n        {\n         if(t_vectors[i] == temp_t[j] && temp_t[j] != -1000)\n           {\n            bool count_ready = false;\n\n            for(ulong n=0; n<v.Size(); n++)\n               if(t_vectors[i] == v[n])\n                  count_ready = true;\n\n            if(!count_ready)\n              {\n               count++;\n               v.Resize(count);\n\n               v[count-1] = t_vectors[i];\n\n               temp_t[j] = -1000; //modify so that it can no more be counted\n              }\n            else\n               break;\n            //Print(\"t vectors vector \",t_vectors);\n           }\n         else\n            continue;\n        }\n     }\n\n   return(v);\n  }\n\n//+------------------------------------------------------------------+\n\nvector CKNNNearestNeighbors::CrossValidation_LOOCV(uint initial_k = 0, uint final_k=1)\n  {\n \n    uint iterations = final_k-initial_k;\n    \n     vector cv(iterations); \n        \n      ulong N = m_rows;\n      \n      matrix OG_Matrix = Matrix; //The original matrix\n      ulong OG_rows = m_rows; //the primarly value of rows\n      vector OG_target = m_target;\n      \n      ulong size = N-1;\n      \n      m_rows = m_rows-1;  //leavo one row out\n      \n      for (uint z = initial_k; z<final_k; z++)\n        { \n          if (iterations>1) k = z;\n               \n          double sum_mse = 0;\n          \n          for (ulong i=0; i<size; i++)\n            { \n               MatrixRemoveRow(Matrix,i);\n               m_target.Resize(m_rows);\n               \n               vector P(1), A = { OG_target[i] }, v;\n               \n                    {\n                        v = OG_Matrix.Row(i);\n                        v.Resize(m_cols-1);\n                        \n                        P[0] = KNNAlgorithm(v);\n                    }\n               \n               \n               if (isdebug)\n                   Print(\"\\n Actual \",A,\" Predicted \",P,\" \",i,\" MSE = \",Mse(A,P),\"\\n\");\n                \n               sum_mse += Mse(A,P);\n               \n               Matrix.Copy(OG_Matrix);\n            }\n         \n         cv[z] = (float)sum_mse/size;\n       } \n      \n      Matrix.Copy(OG_Matrix);\n      m_rows = OG_rows;\n      m_target = OG_target;\n      \n      return (cv);\n  }\n\n//+------------------------------------------------------------------+\n\n\nvoid CKNNNearestNeighbors::MatrixRemoveRow(matrix &mat,ulong row)\n {\n    matrix new_matrix(mat.Rows()-1,mat.Cols()); //Remove the one column\n    \n     for (ulong j=0; j<mat.Cols(); j++)\n      for (ulong i=0, new_rows=0; i<mat.Rows(); i++)\n           {\n               if (i == row) continue; \n               else  \n                 {\n                   new_matrix[new_rows][j] = mat[i][j];\n                 \n                   new_rows++;\n                 }\n           }\n           \n    mat.Copy(new_matrix);\n }\n//+------------------------------------------------------------------+\n\nvoid CKNNNearestNeighbors::VectorRemoveIndex(vector &v, ulong index)\n {\n   vector new_v(v.Size()-1);\n   \n   for (ulong i=0, count = 0; i<v.Size(); i++)\n      if (i == index)\n        {\n          new_v[count] = new_v[i];\n          count++;\n        }\n }\n\n//+------------------------------------------------------------------+\n\ndouble CKNNNearestNeighbors::Mse(vector &A,vector &P)\n {\n   double err = 0;\n   vector c;\n   \n    if (A.Size() != P.Size()) \n      Print(__FUNCTION__,\" Err, A and P vectors not the same in size\");\n    else\n      {\n         ulong size = A.Size();\n         c.Resize(size);\n         \n         c = MathPow(A - P,2); \n         \n         err = (c.Sum()) / size;          \n      }\n    return (err);\n }\n\n//+------------------------------------------------------------------+\nmatrix CKNNNearestNeighbors::ConfusionMatrix(vector &A,vector &P)\n {\n   ulong size = m_classesVector.Size();\n   matrix mat_(size,size);\n   \n   if (A.Size() != P.Size()) \n      Print(\"Cant create confusion matrix | A and P not having the same size \");\n   else\n     {\n      \n         int tn = 0,fn =0,fp =0, tp=0;\n         for (ulong i = 0; i<A.Size(); i++)\n            {              \n               if (A[i]== P[i] && P[i]==m_classesVector[0])\n                  tp++; \n               if (A[i]== P[i] && P[i]==m_classesVector[1])\n                  tn++;\n               if (P[i]==m_classesVector[0] && A[i]==m_classesVector[1])\n                  fp++;\n               if (P[i]==m_classesVector[1] && A[i]==m_classesVector[0])\n                  fn++;\n            }\n            \n       mat_[0][0] = tn; mat_[0][1] = fp;\n       mat_[1][0] = fn; mat_[1][1] = tp;\n\n    }\n     \n   return(mat_);\n    \n }\n//+------------------------------------------------------------------+\nfloat CKNNNearestNeighbors::TrainTest(double train_size=0.700000)\n {\n//--- Split the matrix\n   \n   matrix default_Matrix = Matrix; \n   \n   int train = (int)MathCeil(m_rows*train_size),\n       test  = (int)MathFloor(m_rows*(1-train_size));\n   \n   if (isdebug) printf(\"Train %d test %d\",train,test);\n\n   matrix TrainMatrix(train,m_cols), TestMatrix(test,m_cols);\n   int train_index = 0, test_index =0;\n\n//---\n   \n   for (ulong r=0; r<Matrix.Rows(); r++)\n      {\n         if ((int)r < train)\n           {\n             TrainMatrix.Row(Matrix.Row(r),train_index);\n             train_index++;\n           }\n         else\n           {\n             TestMatrix.Row(Matrix.Row(r),test_index);\n             test_index++;\n           }     \n      }\n\n   if (isdebug) \n    Print(\"TrainMatrix\\n\",TrainMatrix,\"\\nTestMatrix\\n\",TestMatrix);\n   \n//--- Training the Algorithm\n   \n   Matrix.Copy(TrainMatrix); //That's it ???\n   \n//--- Testing the Algorithm\n   \n   vector TestPred(TestMatrix.Rows());\n   vector TargetPred = TestMatrix.Col(m_cols-1);\n   vector v_in = {};\n   \n   for (ulong i=0; i<TestMatrix.Rows(); i++)\n     {\n        v_in = TestMatrix.Row(i);\n        v_in.Resize(v_in.Size()-1); //Remove independent variable\n        \n        TestPred[i] = KNNAlgorithm(v_in);        \n     }\n   \n   matrix cf_m = ConfusionMatrix(TargetPred,TestPred);\n   vector diag = cf_m.Diag();\n   float acc = (float)(diag.Sum()/cf_m.Sum())*100;\n   \n   Print(\"Confusion Matrix\\n\",cf_m,\"\\nAccuracy ------> \",acc,\"%\");\n   \n   return(acc);      \n }\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Sklearn/Tree/README.md",
    "content": "## Decision Trees in MQL5: Classification and Regression\n\nDecision trees are powerful machine learning algorithms that use a tree-like structure to make predictions. They work by splitting the data based on features (independent variables) into increasingly homogeneous subsets, ultimately reaching leaves representing the final prediction. MQL5 offers functionalities for implementing both **classification** and **regression** decision trees through the `tree.mqh` library.\n\n**Decision Tree Theory (Basic Overview):**\n\n1. **Start with the entire dataset at the root node.**\n2. **Choose the feature and threshold that best splits the data into two subsets such that each subset is more homogeneous concerning the target variable (dependent variable).**\n    * For classification, this often involves maximizing information gain or minimizing Gini impurity.\n    * For regression, it involves maximizing variance reduction between the parent node and child nodes.\n3. **Repeat step 2 for each child node recursively until a stopping criterion is met (e.g., reaching a maximum depth, minimum samples per node, or sufficient homogeneity).**\n4. **Assign a prediction value to each leaf node.**\n    * For classification, this is the most frequent class in the leaf node.\n    * For regression, this is the average value of the target variable in the leaf node.\n\n**CDecisionTreeClassifier Class:**\n\nThis class implements a decision tree for classification tasks. It offers the following functionalities:\n\n* `CDecisionTreeClassifier(uint min_samples_split=2, uint max_depth=2, mode mode_=MODE_GINI)` Constructor, allows setting hyperparameters (minimum samples per split, maximum tree depth, and splitting criterion).\n* `~CDecisionTreeClassifier(void)` Destructor.\n* `void fit(const matrix &x, const vector &y)` Trains the model on the provided data (`x` - independent variables, `y` - class labels).\n* `void print_tree(Node *tree, string indent=\" \",string padl=\")` Prints the tree structure in a readable format.\n* `double predict(const vector &x)` Predicts the class label for a new data point (`x`).\n* `vector predict(const matrix &x)` Predicts class labels for multiple new data points (`x`).\n\n**CDecisionTreeRegressor Class:**\n\nThis class inherits from `CDecisionTreeClassifier` and specializes in regression tasks. It overrides specific functions and implements different splitting criteria:\n\n* `CDecisionTreeRegressor(uint min_samples_split=2, uint max_depth=2):` Constructor, allows setting hyperparameters (minimum samples per split and maximum tree depth).\n* `~CDecisionTreeRegressor(void):` Destructor.\n* `void fit(matrix &x, vector &y):` Trains the model on the provided data (`x` - independent variables, `y` - continuous values).\n* `double predict(const vector &x)` Predicts the continuous value for a new data point (`x`).\n\n**Additional Notes:**\n\n* Both classes use internal helper functions for building the tree, calculating splitting criteria (information gain, Gini impurity, variance reduction), and making predictions.\n* The `check_is_fitted` function ensures the model is trained before allowing predictions.\n* Choosing appropriate hyperparameters (especially maximum depth) is crucial to avoid overfitting the model.\n\n**Reference**\n\n[Data Science and Machine Learning (Part 16): A Refreshing Look at Decision Trees](https://www.mql5.com/en/articles/13862)"
  },
  {
    "path": "Sklearn/Tree/tree.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                         tree.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n#include <MALE5\\Utils.mqh>\n\n\n#define log2(leaf_value) MathLog(leaf_value) / MathLog(2)\n\n\nclass Node\n{\n  public:\n    // for decision node\n       \n    uint feature_index;\n    double threshold;\n    double info_gain;\n     \n    // for leaf node\n     \n    double leaf_value;   \n      \n    Node *left_child; //left child Node\n    Node *right_child; //right child Node\n\n    Node() : left_child(NULL), right_child(NULL) {} // default constructor\n\n    Node(int feature_index_, double threshold_=0.0, Node *left_=NULL, Node *right_=NULL, double info_gain_=NULL, double value_=NULL)\n        : left_child(left_), right_child(right_)\n    {\n        this.feature_index = feature_index_;\n        this.threshold = threshold_;\n        this.info_gain = info_gain_;\n        this.leaf_value = value_;\n    }\n    \n   void __Print__()\n    {\n      printf(\"feature_index: %d \\nthreshold: %f \\ninfo_gain: %f \\nleaf_value: %f\",feature_index,threshold, info_gain, leaf_value);\n    }    \n};\n\nstruct split_info\n  {\n   uint feature_index;\n   double threshold;\n   matrix dataset_left,\n          dataset_right;\n   double info_gain;\n  };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nenum mode {MODE_ENTROPY, MODE_GINI};\n\nclass CDecisionTreeClassifier\n  {\nprotected:  \n   \n   Node *build_tree(matrix &data, uint curr_depth=0);\n   double  calculate_leaf_value(vector &Y);\n   \n   bool is_fitted;\n   bool check_is_fitted(string func)\n     {\n       if (!is_fitted)\n         {\n           Print(func,\" Tree not trained, Call fit function first to train the model\");\n           return false;   \n         }\n       return (true);\n     }\n//---\n   \n   uint m_max_depth;\n   uint m_min_samples_split;   \n   mode m_mode;\n   \n   double  gini_index(vector &y);\n   double  entropy(vector &y);\n   double  information_gain(vector &parent, vector &l_child, vector &r_child);\n   \n   \n   split_info  get_best_split(const matrix &data, uint num_features);\n   split_info  split_data(const matrix &data, uint feature_index, double threshold=0.5);\n   \n   double make_predictions(const vector &x, const Node &tree);\n   \n   void delete_tree(Node* node);\n   \n   Node *nodes[]; //Keeping track of all the nodes in a tree\n   \npublic:\n                     Node *root;\n                     \n                     CDecisionTreeClassifier(uint min_samples_split=2, uint max_depth=2, mode mode_=MODE_GINI);\n                    ~CDecisionTreeClassifier(void);\n                    \n                     void fit(const matrix &x, const vector &y);\n                     void print_tree(Node *tree, string indent=\" \",string padl=\"\");\n                     double predict(const vector &x);\n                     vector predict(const matrix &x);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDecisionTreeClassifier::CDecisionTreeClassifier(uint min_samples_split=2, uint max_depth=2, mode mode_=MODE_GINI)\n {\n   m_min_samples_split = min_samples_split;\n   m_max_depth = max_depth;\n   \n   m_mode = mode_;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDecisionTreeClassifier::~CDecisionTreeClassifier(void)\n {   \n   #ifdef DEBUG_MODE\n      Print(__FUNCTION__,\" Deleting Tree nodes =\",nodes.Size());\n   #endif \n   \n   this.delete_tree(root);\n   \n   for (int i=0; i<(int)nodes.Size(); i++)\n     this.delete_tree(nodes[i]);  \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CDecisionTreeClassifier::delete_tree(Node* node)\n {\n    if (CheckPointer(node) != POINTER_INVALID)\n    {\n        delete_tree(node.left_child);\n        delete_tree(node.right_child);\n        delete node;\n    }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::gini_index(vector &y)\n {\n   vector unique = CUtils::Unique_count(y);\n   \n   vector probabilities = unique / (double)y.Size();\n   \n   return 1.0 - MathPow(probabilities, 2).Sum();\n }\n//+------------------------------------------------------------------+\n//|      function to compute entropy                                 |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::entropy(vector &y)\n {    \n   vector class_labels = CUtils::Unique_count(y);\n     \n   vector p_cls = class_labels / double(y.Size());\n  \n   vector entropy = (-1 * p_cls) * log2(p_cls);\n  \n  return entropy.Sum();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::information_gain(vector &parent, vector &l_child, vector &r_child)\n {  \n    double weight_left = l_child.Size() / (double)parent.Size(),\n           weight_right = r_child.Size() / (double)parent.Size();\n    \n    double gain =0;    \n    switch(m_mode)\n      {\n       case  MODE_GINI:\n         gain = gini_index(parent) - ( (weight_left*gini_index(l_child)) + (weight_right*gini_index(r_child)) );\n         break;\n       case MODE_ENTROPY:\n         gain = entropy(parent) - ( (weight_left*entropy(l_child)) + (weight_right*entropy(r_child)) );\n         break;\n      }\n    \n   return gain;\n }\n//+------------------------------------------------------------------+\n//|         function to print the tree                               |\n//+------------------------------------------------------------------+\nvoid CDecisionTreeClassifier::print_tree(Node *tree, string indent=\" \",string padl=\"\")\n  {\n     if (tree.leaf_value != NULL)\n        Print((padl+indent+\": \"),tree.leaf_value); \n     else //if we havent' reached the leaf node keep printing child trees\n       {\n         padl += \" \";\n         \n         Print((padl+indent)+\": X_\",tree.feature_index, \"<=\", tree.threshold, \"?\", tree.info_gain);\n         \n         print_tree(tree.left_child, \"left\",\"--->\"+padl);\n         \n         print_tree(tree.right_child, \"right\",\"--->\"+padl);\n       }\n  }  \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CDecisionTreeClassifier::fit(const matrix &x, const vector &y)\n {   \n   matrix data = CUtils::concatenate(x, y, 1);\n   \n   this.root = this.build_tree(data);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nsplit_info CDecisionTreeClassifier::split_data(const matrix &data, uint feature_index, double threshold=0.5)\n {\n   int left_size=0, right_size =0;\n   vector row = {};\n   \n   split_info split;\n   \n   ulong cols = data.Cols(),\n         rows = data.Rows();\n   \n   split.dataset_left.Resize(0, cols);\n   split.dataset_right.Resize(0, cols);\n   \n   \n    for (ulong i=0; i<rows; i++)\n     {       \n       row = data.Row(i);\n       \n       if (row[feature_index] <= threshold)\n        {\n          left_size++;\n          split.dataset_left.Resize(left_size, cols);\n          split.dataset_left.Row(row, left_size-1); \n        }\n       else\n        {\n         right_size++;\n         split.dataset_right.Resize(right_size, cols);\n         split.dataset_right.Row(row, right_size-1);         \n        }\n     }\n     \n   return split;\n }\n//+------------------------------------------------------------------+\n//|      Return the Node for the best split                          |\n//+------------------------------------------------------------------+\nsplit_info CDecisionTreeClassifier::get_best_split(const matrix &data, uint num_features)\n  {\n  \n   double max_info_gain = -DBL_MAX;\n   vector feature_values = {};\n   vector left_v={}, right_v={}, y_v={};\n   \n//---\n   \n   split_info best_split;\n   split_info split;\n   \n   for (int i=0; i<(int)num_features; i++)\n     {\n       feature_values = data.Col(i);\n       vector possible_thresholds = CUtils::Unique(feature_values);\n              \n        if (possible_thresholds.Size() <= 1)\n           continue; // Skip this feature as it won't provide meaningful splits\n             \n       //---\n             \n         for (int j=0; j<(int)possible_thresholds.Size(); j++)\n            {                            \n              split = this.split_data(data, i, possible_thresholds[j]);\n              \n              if (split.dataset_left.Rows()>0 && split.dataset_right.Rows() > 0)\n                {\n                  y_v = data.Col(data.Cols()-1);\n                  right_v = split.dataset_right.Col(split.dataset_right.Cols()-1);\n                  left_v = split.dataset_left.Col(split.dataset_left.Cols()-1);\n                  \n                  double curr_info_gain = this.information_gain(y_v, left_v, right_v);\n                                    \n                  if (curr_info_gain > max_info_gain)\n                    {             \n                      #ifdef DEBUG_MODE\n                        printf(\"    split left: [%dx%d] split right: [%dx%d] curr_info_gain: %f max_info_gain: %f\",split.dataset_left.Rows(),split.dataset_left.Cols(),split.dataset_right.Rows(),split.dataset_right.Cols(),curr_info_gain,max_info_gain);\n                      #endif \n                        \n                      best_split.feature_index = i;\n                      best_split.threshold = possible_thresholds[j];\n                      best_split.dataset_left = split.dataset_left;\n                      best_split.dataset_right = split.dataset_right;\n                      best_split.info_gain = curr_info_gain;\n                      \n                      max_info_gain = curr_info_gain;\n                    }\n                }\n            }    \n     }\n     \n    return best_split;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nNode *CDecisionTreeClassifier::build_tree(matrix &data, uint curr_depth=0)\n {\n    matrix X;\n    vector Y;\n    \n         \n    if (!CUtils::XandYSplitMatrices(data,X,Y)) //Split the input matrix into feature matrix X and target vector Y.    \n      {\n         #ifdef DEBUG_MODE\n            printf(\"%s Line %d Failed to build a tree Data Empty\",__FUNCTION__,__LINE__);\n         #endif \n         \n         return NULL; //return null pointer\n      }\n    \n    is_fitted = true;\n     \n    ulong samples = X.Rows(), features = X.Cols(); //Get the number of samples and features in the dataset.\n        \n    ArrayResize(nodes, nodes.Size()+1); //Append the nodes to memory\n    Node *left_child, *right_child;\n            \n    if (samples >= m_min_samples_split && curr_depth<=m_max_depth)\n      {\n         split_info best_split = this.get_best_split(data, (uint)features);\n         \n         #ifdef DEBUG_MODE\n             Print(__FUNCTION__,\" | \",__LINE__,\"\\nbest_split left: [\",best_split.dataset_left.Rows(),\"x\",best_split.dataset_left.Cols(),\"]\\nbest_split right: [\",best_split.dataset_right.Rows(),\"x\",best_split.dataset_right.Cols(),\"]\\nfeature_index: \",best_split.feature_index,\"\\nInfo gain: \",best_split.info_gain,\"\\nThreshold: \",best_split.threshold);\n         #endif \n                  \n         if (best_split.info_gain > 0)\n           {\n             left_child = this.build_tree(best_split.dataset_left, curr_depth+1);\n             right_child = this.build_tree(best_split.dataset_right, curr_depth+1);\n                      \n             nodes[nodes.Size()-1] = new Node(best_split.feature_index,best_split.threshold,left_child,right_child,best_split.info_gain);  \n             return nodes[nodes.Size()-1];\n           }\n      }      \n     \n     nodes[nodes.Size()-1] = new Node();\n     nodes[nodes.Size()-1].leaf_value = this.calculate_leaf_value(Y);\n     \n     return nodes[nodes.Size()-1];\n }\n//+------------------------------------------------------------------+\n//|   returns the element from Y that has the highest count,         |\n//|  effectively finding the most common element in the list.        |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::calculate_leaf_value(vector &Y)\n {   \n   vector uniques_count = CUtils::Unique_count(Y);\n   vector unique = CUtils::Unique(Y);\n   \n   return unique[uniques_count.ArgMax()];\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::make_predictions(const vector &x, const Node &tree)\n {   \n   if (!check_is_fitted(__FUNCTION__))\n     return 0;\n   \n   //if (CheckPointer(tree)=POINTER_INVALID)\n     \n    if (tree.leaf_value != NULL) //This is a leaf_value\n      return tree.leaf_value;\n    \n    #ifdef DEBUG_MODE\n      printf(\"Tree.threshold %f tree.feature_index %d leaf_value %f\",tree.threshold,tree.feature_index,tree.leaf_value);\n    #endif \n    \n    if (tree.feature_index>=x.Size())\n      return tree.leaf_value;\n          \n    double feature_value = x[tree.feature_index];\n    double pred = 0;\n    \n    if (feature_value <= tree.threshold)\n      {\n       if (CheckPointer(tree.left_child)!=POINTER_INVALID)\n          pred = this.make_predictions(x, tree.left_child);  \n      }\n    else\n     {\n       if (CheckPointer(tree.right_child)!=POINTER_INVALID)\n         pred = this.make_predictions(x, tree.right_child);\n     }\n     \n   return pred;\n }\n//+------------------------------------------------------------------+\n//|      Commonly used for making predictions in REAL-TIME           |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeClassifier::predict(const vector &x)\n {\n   if (!check_is_fitted(__FUNCTION__))\n     return 0;\n          \n   return this.make_predictions(x, this.root);\n }\n//+------------------------------------------------------------------+\n//|   Commonly used for making predictions in TRAIN-TEST             |\n//+------------------------------------------------------------------+\nvector CDecisionTreeClassifier::predict(const matrix &x)\n {\n    vector ret(x.Rows());\n \n   if (!check_is_fitted(__FUNCTION__))\n     return ret;\n        \n    for (ulong i=0; i<x.Rows(); i++)\n       ret[i] = this.predict(x.Row(i));\n       \n   return ret;\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CDecisionTreeRegressor: public CDecisionTreeClassifier\n  {\nprivate:\n                     double  calculate_leaf_value(vector &Y);\n                     split_info  get_best_split(matrix &data, uint num_features);\n                     double variance_reduction(vector &parent, vector &l_child, vector &r_child);\n                     \n                     Node *build_tree(matrix &data, uint curr_depth=0);\npublic:\n                     CDecisionTreeRegressor(uint min_samples_split=2, uint max_depth=2);\n                    ~CDecisionTreeRegressor(void);\n                    \n                     void fit(matrix &x, vector &y);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDecisionTreeRegressor::CDecisionTreeRegressor(uint min_samples_split=2, uint max_depth=2):CDecisionTreeClassifier(min_samples_split, max_depth)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCDecisionTreeRegressor::~CDecisionTreeRegressor(void)\n {\n \n }\n//+------------------------------------------------------------------+\n//|         function to compute variance reduction                   |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeRegressor::variance_reduction(vector &parent, vector &l_child, vector &r_child)\n {\n    double weight_l = l_child.Size() / (double)parent.Size(),\n           weight_r = r_child.Size() / (double)parent.Size();\n     \n    return parent.Var() - ((weight_l * l_child.Var()) + (weight_r * r_child.Var()));\n }\n//+------------------------------------------------------------------+\n//|      Return the Node for the best split                          |\n//+------------------------------------------------------------------+\nsplit_info CDecisionTreeRegressor::get_best_split(matrix &data, uint num_features)\n  {\n   double max_info_gain = -DBL_MAX;\n   vector feature_values = {};\n   vector left_v={}, right_v={}, y_v={};\n   \n//---\n   \n   split_info best_split;\n   split_info split;\n   \n   for (uint i=0; i<num_features; i++)\n     {\n       feature_values = data.Col(i);\n       vector possible_thresholds = CUtils::Unique(feature_values);\n                  \n         for (uint j=0; j<possible_thresholds.Size(); j++)\n            {              \n              split = this.split_data(data, i, possible_thresholds[j]);\n              \n              if (split.dataset_left.Rows()>0 && split.dataset_right.Rows() > 0)\n                {\n                  y_v = data.Col(data.Cols()-1);\n                  right_v = split.dataset_right.Col(split.dataset_right.Cols()-1);\n                  left_v = split.dataset_left.Col(split.dataset_left.Cols()-1);\n                  \n                  double curr_info_gain = this.variance_reduction(y_v, left_v, right_v);\n                                    \n                  if (curr_info_gain > max_info_gain)\n                    {             \n                      #ifdef DEBUG_MODE\n                        printf(__FUNCTION__,\" | \",__LINE__,\"\\nsplit left: [%dx%d] split right: [%dx%d] curr_info_gain: %f max_info_gain: %f\",split.dataset_left.Rows(),split.dataset_left.Cols(),split.dataset_right.Rows(),split.dataset_right.Cols(),curr_info_gain,max_info_gain);\n                      #endif \n                      \n                      best_split.feature_index = i;\n                      best_split.threshold = possible_thresholds[j];\n                      best_split.dataset_left = split.dataset_left;\n                      best_split.dataset_right = split.dataset_right;\n                      best_split.info_gain = curr_info_gain;\n                      \n                      max_info_gain = curr_info_gain;\n                    }\n                }\n            }    \n     }\n     \n    return best_split;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nNode *CDecisionTreeRegressor::build_tree(matrix &data, uint curr_depth=0)\n {\n    matrix X;\n    vector Y;\n      \n    if (!CUtils::XandYSplitMatrices(data,X,Y)) //Split the input matrix into feature matrix X and target vector Y.    \n      {\n         #ifdef DEBUG_MODE \n           printf(\"%s Line %d Failed to build a tree Data Empty\",__FUNCTION__,__LINE__);\n         #endif \n         \n         return NULL; //Return a NULL pointer\n      }\n      \n    ulong samples = X.Rows(), features = X.Cols(); //Get the number of samples and features in the dataset.\n        \n    ArrayResize(nodes, nodes.Size()+1); //Append the nodes to memory\n    Node *left_child, *right_child;\n            \n    if (samples >= m_min_samples_split && curr_depth<=m_max_depth)\n      {\n         split_info best_split = this.get_best_split(data, (uint)features);\n         \n         #ifdef DEBUG_MODE\n           Print(__FUNCTION__,\" | \",__LINE__,\"\\nbest_split left: [\",best_split.dataset_left.Rows(),\"x\",best_split.dataset_left.Cols(),\"]\\nbest_split right: [\",best_split.dataset_right.Rows(),\"x\",best_split.dataset_right.Cols(),\"]\\nfeature_index: \",best_split.feature_index,\"\\nInfo gain: \",best_split.info_gain,\"\\nThreshold: \",best_split.threshold);\n         #endif \n                  \n         if (best_split.info_gain > 0)\n           {\n             left_child = this.build_tree(best_split.dataset_left, curr_depth+1);\n             right_child = this.build_tree(best_split.dataset_right, curr_depth+1);\n                      \n             nodes[nodes.Size()-1] = new Node(best_split.feature_index,best_split.threshold,left_child,right_child,best_split.info_gain);  \n             return nodes[nodes.Size()-1];\n           }\n      }      \n     \n     nodes[nodes.Size()-1] = new Node();\n     nodes[nodes.Size()-1].leaf_value = this.calculate_leaf_value(Y);\n     \n     return nodes[nodes.Size()-1];\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CDecisionTreeRegressor::fit(matrix &x, vector &y)\n {\n   matrix data = CUtils::concatenate(x, y, 1);\n      \n   this.root = this.build_tree(data);\n   \n   is_fitted = true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CDecisionTreeRegressor::calculate_leaf_value(vector &Y)\n {\n   return Y.Mean();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Sklearn/metrics.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                      metrics.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\Numpy\\Numpy.mqh>\n#include <MALE5\\MqPlotLib\\plots.mqh>\n\nstruct roc_curve_struct\n {\n   vector TPR,\n          FPR, \n          Thresholds;\n };\n\nstruct confusion_matrix_struct\n { \n   matrix MATRIX;\n   vector CLASSES;\n   vector TP, \n          TN, \n          FP, \n          FN;\n };\n \nenum regression_metrics\n{\n   METRIC_R_SQUARED,   // R-squared\n   METRIC_ADJUSTED_R,  // Adjusted R-squared\n   METRIC_RSS,         // Residual Sum of Squares\n   METRIC_MSE,         // Mean Squared Error\n   METRIC_RMSE,        // Root Mean Squared Error\n   METRIC_MAE          // Mean Absolute Error\n};\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass Metrics\n  {\nprotected:   \n  \n   static int SearchPatterns(const vector &True, int value_A, const vector &B, int value_B);\n   static confusion_matrix_struct confusion_matrix(const vector &True, const vector &Preds);\n   \npublic:\n\n   Metrics(void);\n   ~Metrics(void);\n\n   //--- Regression metrics\n\n   static double r_squared(const vector &True, const vector &Pred);\n   static double adjusted_r(const vector &True, const vector &Pred, uint indep_vars = 1);\n\n   static double rss(const vector &True, const vector &Pred);\n   static double mse(const vector &True, const vector &Pred);\n   static double rmse(const vector &True, const vector &Pred);\n   static double mae(const vector &True, const vector &Pred);\n   \n   static double RegressionMetric(const vector &True, const vector &Pred, regression_metrics METRIC_);\n\n   //--- Classification metrics\n\n   static double accuracy_score(const vector &True, const vector &Pred);\n   \n   static vector accuracy(const vector &True, const vector &Preds);\n   static vector precision(const vector &True, const vector &Preds);\n   static vector recall(const vector &True, const vector &Preds);\n   static vector f1_score(const vector &True, const vector &Preds);\n   static vector specificity(const vector &True, const vector &Preds);\n   \n   static roc_curve_struct roc_curve(const vector &True, const vector &Preds, bool show_roc_curve=false);\n   static void classification_report(const vector &True, const vector &Pred, bool show_roc_curve=false);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nMetrics::Metrics(void)\n  {\n\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nMetrics::~Metrics(void)\n  {\n\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::r_squared(const vector &True, const vector &Pred)\n  {\n   return(Pred.RegressionMetric(True, REGRESSION_R2));\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::adjusted_r(const vector &True, const vector &Pred, uint indep_vars = 1)\n  {\n   if(True.Size() != Pred.Size())\n     {\n      Print(__FUNCTION__, \" Vector True and P are not equal in size \");\n      return(0);\n     }\n\n   double r2 = r_squared(True, Pred);\n   ulong N = Pred.Size();\n\n   return(1 - ((1 - r2) * (N - 1)) / (N - indep_vars - 1));\n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nconfusion_matrix_struct Metrics::confusion_matrix(const vector &True, const vector &Preds)\n {\n  confusion_matrix_struct confusion_matrix; \n   \n  vector classes = CNumpy::unique(True).unique;\n  confusion_matrix.CLASSES = classes;\n  \n//--- Fill the confusion matrix\n   \n   matrix MATRIX(classes.Size(), classes.Size());\n   MATRIX.Fill(0.0);\n   \n   for(ulong i = 0; i < classes.Size(); i++)\n      for(ulong j = 0; j < classes.Size(); j++)\n         MATRIX[i][j] = SearchPatterns(True, (int)classes[i], Preds, (int)classes[j]);\n   \n   confusion_matrix.MATRIX = MATRIX;\n   confusion_matrix.TP = MATRIX.Diag();\n   confusion_matrix.FP = MATRIX.Sum(0) - confusion_matrix.TP;\n   confusion_matrix.FN = MATRIX.Sum(1) - confusion_matrix.TP;\n   confusion_matrix.TN = MATRIX.Sum() - (confusion_matrix.TP + confusion_matrix.FP + confusion_matrix.FN);\n     \n   return confusion_matrix;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector Metrics::accuracy(const vector &True,const vector &Preds)\n {\n  confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n  \n  return (conf_m.TP + conf_m.TN) / conf_m.MATRIX.Sum();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector Metrics::precision(const vector &True,const vector &Preds)\n {\n   confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n\n   return conf_m.TP / (conf_m.TP + conf_m.FP + DBL_EPSILON); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector Metrics::f1_score(const vector &True,const vector &Preds)\n {\n   vector precision = precision(True, Preds);\n   vector recall = recall(True, Preds);\n   \n   return 2 * precision * recall / (precision + recall + DBL_EPSILON); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector Metrics::recall(const vector &True,const vector &Preds)\n {\n   confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n\n   return conf_m.TP / (conf_m.TP + conf_m.FN + DBL_EPSILON); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector Metrics::specificity(const vector &True,const vector &Preds)\n {\n   confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n\n   return conf_m.TN / (conf_m.TN + conf_m.FP + DBL_EPSILON); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nroc_curve_struct Metrics::roc_curve(const vector &True,const vector &Preds, bool show_roc_curve=false)\n {\n   roc_curve_struct roc;\n   confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n   \n   roc.TPR = recall(True, Preds);\n   roc.FPR = conf_m.FP / (conf_m.FP + conf_m.TN + DBL_EPSILON);\n   \n   if (show_roc_curve)\n   {\n      CPlots plt;\n      plt.Plot(\"Roc Curve\",roc.FPR,roc.TPR,\"roc_curve\",\"False Positive Rate(FPR)\",\"True Positive Rate(TPR)\");\n      \n      while (MessageBox(\"Close or Cancel ROC CURVE to proceed\",\"Roc Curve\",MB_OK)<0)\n       Sleep(1);\n   } \n   \n   return roc;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::accuracy_score(const vector &True, const vector &Preds)\n  {\n   confusion_matrix_struct conf_m = confusion_matrix(True, Preds);\n   \n   return conf_m.MATRIX.Diag().Sum() / (conf_m.MATRIX.Sum() + DBL_EPSILON);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid Metrics::classification_report(const vector &True, const vector &Pred, bool show_roc_curve=false)\n  {\n  \n  vector accuracy = accuracy(True, Pred);\n  vector precision = precision(True, Pred);\n  vector specificity = specificity(True, Pred);\n  vector recall = recall(True, Pred);\n  vector f1_score = f1_score(True, Pred); \n  \n  double acc = accuracy_score(True, Pred);\n  \n  confusion_matrix_struct conf_m = confusion_matrix(True, Pred);\n  \n//--- support\n   \n   ulong size = conf_m.MATRIX.Rows();\n   \n   vector support(size);\n   \n   for(ulong i = 0; i < size; i++)\n      support[i] = NormalizeDouble(MathIsValidNumber(conf_m.MATRIX.Row(i).Sum()) ? conf_m.MATRIX.Row(i).Sum() : 0, 8);\n\n   int total_size = (int)conf_m.MATRIX.Sum();\n\n//--- Avg and w avg\n   \n   vector avg, w_avg;\n   avg.Resize(5);\n   w_avg.Resize(5);\n\n   avg[0] = precision.Mean();\n\n   avg[1] = recall.Mean();\n   avg[2] = specificity.Mean();\n   avg[3] = f1_score.Mean();\n\n   avg[4] = total_size;\n\n//--- w avg\n\n   vector support_prop = support / double(total_size + 1e-10);\n\n   vector c = precision * support_prop;\n   w_avg[0] = c.Sum();\n\n   c = recall * support_prop;\n   w_avg[1] = c.Sum();\n\n   c = specificity * support_prop;\n   w_avg[2] = c.Sum();\n\n   c = f1_score * support_prop;\n   w_avg[3] = c.Sum();\n\n   w_avg[4] = (int)total_size;\n\n//--- Report\n\n      string report = \"\\n[CLS] \\t\\t\\t\\t\\t\\t\\tprecision \\trecall \\tspecificity \\tf1 score \\tsupport\";\n\n      for(ulong i = 0; i < size; i++)\n        {\n         report += \"\\n\\t\\t[\" + string(conf_m.CLASSES[i])+\"]\\t\\t\\t\";\n         //for (ulong j=0; j<3; j++)\n\n         report += StringFormat(\"\\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t %d\", precision[i], recall[i], specificity[i], f1_score[i], (int)support[i]);\n        }\n      \n      report += \"\\n\";\n      report += StringFormat(\"\\naccuracy\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t %d\",acc,(int)conf_m.MATRIX.Sum());\n      \n      report += StringFormat(\"\\naverage\\t\\t\\t\\t\\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t %d\", avg[0], avg[1], avg[2], avg[3], (int)avg[4]);\n      report += StringFormat(\"\\nWeighed avg\\t\\t\\t \\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t\\t %.2f \\t\\t\\t\\t %d\", w_avg[0], w_avg[1], w_avg[2], w_avg[3], (int)w_avg[4]);\n\n      Print(\"Confusion Matrix\\n\", conf_m.MATRIX);\n      Print(\"\\nClassification Report\\n\", report);\n      \n      roc_curve(True, Pred, show_roc_curve);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::rss(const vector &True, const vector &Pred)\n  {\n   vector c = True - Pred;\n   c = MathPow(c, 2);\n\n   return (c.Sum());\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::mse(const vector &True, const vector &Pred)\n  {\n   vector c = True - Pred;\n   c = MathPow(c, 2);\n\n   return(c.Sum() / c.Size());\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint Metrics::SearchPatterns(const vector &True, int value_A, const vector &B, int value_B)\n  {\n   int count=0;\n   \n   for(ulong i = 0; i < True.Size(); i++)\n      if(True[i] == value_A && B[i] == value_B)\n         count++;\n\n   return count;\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::rmse(const vector &True, const vector &Pred)\n  {\n   return Pred.RegressionMetric(True, REGRESSION_RMSE);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::mae(const vector &True, const vector &Pred)\n  {\n   return Pred.RegressionMetric(True, REGRESSION_MAE);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble Metrics::RegressionMetric(const vector &True,const vector &Pred,regression_metrics METRIC_)\n {\n  double err = 0;\n  \n  switch (METRIC_)\n   {\n     case METRIC_MSE:\n         err = mse(True, Pred);\n         break;\n     case METRIC_RMSE:\n         err = rmse(True, Pred);\n         break;\n     case METRIC_MAE:\n         err = mae(True, Pred);\n         break;\n     case METRIC_RSS:\n         err = rss(True, Pred);\n         break;\n     case METRIC_R_SQUARED:\n         err = r_squared(True, Pred);\n         break;\n     case METRIC_ADJUSTED_R:\n         err = adjusted_r(True, Pred);\n         break;\n     default:\n         break;\n   }\n\n  return err;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Stats Models/ADF.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                          ADF.mqh |\n//|                                  Copyright 2023, MetaQuotes Ltd. |\n//|                                             https://www.mql5.com |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, MetaQuotes Ltd.\"\n#property link      \"https://www.mql5.com\"\n#include<Math\\Stat\\Math.mqh>\n#include<Math\\Alglib\\specialfunctions.mqh>\n#include \"OLS.mqh\"\n#define SQRTEPS 1.4901161193847656e-08\n//+------------------------------------------------------------------+\n//| Information criterion                                            |\n//+------------------------------------------------------------------+\nenum ENUM_INFO_CRIT\n  {\n   INFO_NONE=0,\n   INFO_AIC,\n   INFO_BIC\n  };\n//+------------------------------------------------------------------+\n//| Options for  trimming invalid observations                       |\n//+------------------------------------------------------------------+\nenum ENUM_TRIM\n  {\n   TRIM_NONE=0,\n   TRIM_FORWARD,\n   TRIM_BACKWARD,\n   TRIM_BOTH\n  };\n//+------------------------------------------------------------------+\n//| options for how to handle original data set                      |\n//+------------------------------------------------------------------+\n\nenum ENUM_ORIGINAL\n  {\n   ORIGINAL_EX=0,\n   ORIGINAL_IN,\n   ORIGINAL_SEP\n  };\n//+------------------------------------------------------------------+\n//| Constant and trend used in regression model                      |\n//+------------------------------------------------------------------+\n\nenum ENUM_TREND\n  {\n   TREND_NONE=0,\n   TREND_CONST_ONLY,\n   TREND_LINEAR_ONLY,\n   TREND_LINEAR_CONST,\n   TREND_QUAD_LINEAR_CONST\n  };\n//+------------------------------------------------------------------+\n//| Options for how to handle existing constants                     |\n//+------------------------------------------------------------------+\n\nenum ENUM_HAS_CONST\n  {\n   HAS_CONST_RAISE=0,\n   HAS_CONST_SKIP,\n   HAS_CONST_ADD\n  };\n//+------------------------------------------------------------------+\n//|helper function : adds selected trend and\\or constant             |\n//+------------------------------------------------------------------+\nbool addtrend(matrix &in,matrix &out,ENUM_TREND trend=TREND_CONST_ONLY, bool prepend=false, ENUM_HAS_CONST has_const=HAS_CONST_SKIP)\n  {\n//---\n   ulong trendorder=0;\n//---\n   if(trend==TREND_NONE)\n      return out.Copy(in);\n//---\n   if(trend==TREND_CONST_ONLY)\n      trendorder = 0;\n   else\n      if(trend==TREND_LINEAR_CONST || trend==TREND_LINEAR_ONLY)\n         trendorder = 1;\n      else\n         if(trend==TREND_QUAD_LINEAR_CONST)\n            trendorder = 2;\n//---\n   ulong nobs = in.Rows();\n//---\n   vector tvector,temp;\n//---\n   if(!tvector.Resize(nobs) || !temp.Resize(nobs))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n      return false;\n     }\n//---\n   double sequence[];\n//---\n   if(!MathSequence(1,nobs,1,sequence) || !tvector.Assign(sequence))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Error \", ::GetLastError());\n      return false;\n     }\n//---\n   matrix trendmat;\n//---\n   if(!trendmat.Resize(tvector.Size(),(trend==TREND_LINEAR_ONLY)?1:trendorder+1))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n      return false;\n     }\n//---\n   for(ulong i = 0; i<trendmat.Cols(); i++)\n     {\n      temp = MathPow(tvector,(trend==TREND_LINEAR_ONLY)?double(i+1):double(i));\n      if(!trendmat.Col(temp,i))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Matrix Assign Error \", ::GetLastError());\n         return false;\n        }\n     }\n//---\n   vector ptp = in.Ptp(0);\n//---\n   if(!ptp.Min())\n     {\n      if(has_const==HAS_CONST_RAISE)\n        {\n         Print(\"Input matrix contains one or more constant columns\");\n         return false;\n        }\n      if(has_const==HAS_CONST_SKIP)\n        {\n         matrix vsplitted[];\n\n         ulong parts[] = {1,trendmat.Cols()-1};\n\n         trendmat.Vsplit(parts,vsplitted);\n\n         if(!trendmat.Copy(vsplitted[1]))\n           {\n            Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n            return false;\n           }\n        }\n     }\n//---\n   int order = (prepend)?1:-1;\n//---\n   if(!out.Resize(trendmat.Rows(),trendmat.Cols()+in.Cols()))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n      return false;\n     }\n//---\n   ulong j = 0;\n   matrix mtemp;\n//---\n   if(prepend)\n      mtemp.Copy(trendmat);\n   else\n      mtemp.Copy(in);\n//---\n   for(j = 0; j<mtemp.Cols(); j++)\n     {\n      vector col = mtemp.Col(j);\n\n      if(!out.Col(col,j))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Matrix Assign Error \", ::GetLastError());\n         return false;\n        }\n\n     }\n//---\n   ulong minus = j;\n//---\n   if(prepend)\n      mtemp.Copy(in);\n   else\n      mtemp.Copy(trendmat);\n//---\n   for(; j<out.Cols(); j++)\n     {\n      vector col = mtemp.Col(j-minus);\n\n      if(!out.Col(col,j))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Matrix Assign Error \", ::GetLastError());\n         return false;\n        }\n\n     }\n//---\n   return true;\n//---\n  }\n\n//+----------------------------------------------------------------------+\n//| calculates MacKinnon's approximate p-value for a given test statistic|\n//+----------------------------------------------------------------------+\ndouble mackinnonp(double teststat, ENUM_TREND trend = TREND_CONST_ONLY,ulong nseries = 1, uint lags =0)\n  {\n   vector small_scaling =  {1, 1, 1e-2};\n   vector large_scaling =  {1, 1e-1, 1e-1, 1e-2};\n\n   double tau_star_nc []= {-1.04, -1.53, -2.68, -3.09, -3.07, -3.77};\n   double tau_min_nc []= {-19.04, -19.62, -21.21, -23.25, -21.63, -25.74};\n   double tau_max_nc []= {double(\"inf\"), 1.51, 0.86, 0.88, 1.05, 1.24};\n   double tau_star_c []= {-1.61, -2.62, -3.13, -3.47, -3.78, -3.93};\n   double tau_min_c []= {-18.83, -18.86, -23.48, -28.07, -25.96, -23.27};\n   double tau_max_c []= {2.74, 0.92, 0.55, 0.61, 0.79, 1};\n   double tau_star_ct []= {-2.89, -3.19, -3.50, -3.65, -3.80, -4.36};\n   double tau_min_ct []= {-16.18, -21.15, -25.37, -26.63, -26.53, -26.18};\n   double tau_max_ct []= {0.7, 0.63, 0.71, 0.93, 1.19, 1.42};\n   double tau_star_ctt []= {-3.21, -3.51, -3.81, -3.83, -4.12, -4.63};\n   double tau_min_ctt []= {-17.17, -21.1, -24.33, -24.03, -24.33, -28.22};\n   double tau_max_ctt []= {0.54, 0.79, 1.08, 1.43, 3.49, 1.92};\n\n   double tau_nc_smallp [][3]=\n     {\n        {0.6344, 1.2378, 3.2496},\n        {1.9129, 1.3857, 3.5322},\n        {2.7648, 1.4502, 3.4186},\n        {3.4336, 1.4835, 3.19},\n        {4.0999, 1.5533, 3.59},\n        {4.5388, 1.5344, 2.9807}\n     };\n\n   double tau_c_smallp [][3]=\n     {\n        {2.1659, 1.4412, 3.8269},\n        {2.92, 1.5012, 3.9796},\n        {3.4699, 1.4856, 3.164},\n        {3.9673, 1.4777, 2.6315},\n        {4.5509, 1.5338, 2.9545},\n        {5.1399, 1.6036, 3.4445}\n     };\n\n   double tau_ct_smallp [][3]=\n     {\n        {3.2512, 1.6047, 4.9588},\n        {3.6646, 1.5419, 3.6448},\n        {4.0983, 1.5173, 2.9898},\n        {4.5844, 1.5338, 2.8796},\n        {5.0722, 1.5634, 2.9472},\n        {5.53, 1.5914, 3.0392}\n     };\n\n   double tau_ctt_smallp [][3]=\n     {\n        {4.0003, 1.658, 4.8288},\n        {4.3534, 1.6016, 3.7947},\n        {4.7343, 1.5768, 3.2396},\n        {5.214, 1.6077, 3.3449},\n        {5.6481, 1.6274, 3.3455},\n        {5.9296, 1.5929, 2.8223}\n     };\n\n\n\n   double tau_nc_largep [][4]=\n     {\n        {0.4797, 9.3557, -0.6999, 3.3066},\n        {1.5578, 8.558, -2.083, -3.3549},\n        {2.2268, 6.8093, -3.2362, -5.4448},\n        {2.7654, 6.4502, -3.0811, -4.4946},\n        {3.2684, 6.8051, -2.6778, -3.4972},\n        {3.7268, 7.167, -2.3648, -2.8288}\n     };\n\n   double tau_c_largep [][4]=\n     {\n        {1.7339, 9.3202, -1.2745, -1.0368},\n        {2.1945, 6.4695, -2.9198, -4.2377},\n        {2.5893, 4.5168, -3.6529, -5.0074},\n        {3.0387, 4.5452, -3.3666, -4.1921},\n        {3.5049, 5.2098, -2.9158, -3.3468},\n        {3.9489, 5.8933, -2.5359, -2.721}\n     };\n\n   double tau_ct_largep [][4]=\n     {\n        {2.5261, 6.1654, -3.7956, -6.0285},\n        {2.85, 5.272, -3.6622, -5.1695},\n        {3.221, 5.255, -3.2685, -4.1501},\n        {3.652, 5.9758, -2.7483, -3.2081},\n        {4.0712, 6.6428, -2.3464, -2.546},\n        {4.4735, 7.1757, -2.0681, -2.1196}\n     };\n\n   double tau_ctt_largep [][4]=\n     {\n        {3.0778, 4.9529, -4.1477, -5.9359},\n        {3.4713, 5.967, -3.2507, -4.2286},\n        {3.8637, 6.7852, -2.6286, -3.1381},\n        {4.2736, 7.6199, -2.1534, -2.4026},\n        {4.6679, 8.2618, -1.822, -1.9147},\n        {5.0009, 8.3735, -1.6994, -1.6928}\n     };\n\n\n   vector maxstat,minstat,starstat;\n   matrix tau_smallps, tau_largeps;\n\n   switch(trend)\n     {\n      case TREND_NONE:\n         if(!maxstat.Assign(tau_max_nc) ||\n            !minstat.Assign(tau_min_nc) ||\n            !starstat.Assign(tau_star_nc)||\n            !tau_smallps.Assign(tau_nc_smallp)||\n            !tau_largeps.Assign(tau_nc_largep))\n           {\n            Print(\"assignment error :\", GetLastError());\n            return double(\"inf\");\n           }\n         else\n            break;\n      case TREND_CONST_ONLY:\n         if(!maxstat.Assign(tau_max_c) ||\n            !minstat.Assign(tau_min_c) ||\n            !starstat.Assign(tau_star_c)||\n            !tau_smallps.Assign(tau_c_smallp)||\n            !tau_largeps.Assign(tau_c_largep))\n           {\n            Print(\"assignment error :\", GetLastError());\n            return double(\"inf\");\n           }\n         else\n            break;\n      case TREND_LINEAR_CONST:\n         if(!maxstat.Assign(tau_max_ct) ||\n            !minstat.Assign(tau_min_ct) ||\n            !starstat.Assign(tau_star_ct)||\n            !tau_smallps.Assign(tau_ct_smallp)||\n            !tau_largeps.Assign(tau_ct_largep))\n           {\n            Print(\"assignment error :\", GetLastError());\n            return double(\"inf\");\n           }\n         else\n            break;\n      case TREND_QUAD_LINEAR_CONST:\n         if(!maxstat.Assign(tau_max_ctt) ||\n            !minstat.Assign(tau_min_ctt) ||\n            !starstat.Assign(tau_star_ctt)||\n            !tau_smallps.Assign(tau_ctt_smallp)||\n            !tau_largeps.Assign(tau_ctt_largep))\n           {\n            Print(\"assignment error :\", GetLastError());\n            return double(\"inf\");\n           }\n         else\n            break;\n      default:\n         Print(__FUNCTION__,\" Error invalid input for trend argument\");\n         return double(\"nan\");\n     }\n\n   if(teststat>maxstat[nseries-1])\n      return 1.0;\n   else\n      if(teststat<minstat[nseries-1])\n         return 0.0;\n\n\n   vector tau_coef;\n\n   if(teststat<=starstat[nseries-1])\n      tau_coef = small_scaling*(tau_smallps.Row(nseries-1));\n   else\n      tau_coef = large_scaling*(tau_largeps.Row(nseries-1));\n\n\n   double rv,tau[];\n\n   ArrayResize(tau,int(tau_coef.Size()));\n\n   for(ulong i=0; i<tau_coef.Size(); i++)\n      tau[i]=tau_coef[tau_coef.Size()-1-i];\n\n   rv=polyval(tau,teststat);\n\n   return CNormalDistr::NormalCDF(rv);\n  }\n\n//+------------------------------------------------------------------+\n//| helper function : evaluates a polynomial                         |\n//+------------------------------------------------------------------+\ndouble polyval(double &in_array[], double coeff)\n  {\n   double retv = 0;\n\n   for(uint i = 0; i<in_array.Size(); i++)\n      retv = retv * coeff + in_array[i];\n\n   return retv;\n  }\n//+------------------------------------------------------------------+\n//|Computes critical values                                          |\n//+------------------------------------------------------------------+\nvector mackinnoncrit(ulong nseries = 1,ENUM_TREND trend = TREND_CONST_ONLY, ulong num_obs=ULONG_MAX)\n  {\n   matrix tau_nc_2010 [] = {{\n           {-2.56574, -2.2358, -3.627, 0},  // N [] = 1\n           {-1.94100, -0.2686, -3.365, 31.223},\n           {-1.61682, 0.2656, -2.714, 25.364}\n        }\n     };\n\n   matrix tau_c_2010 [] =\n     {\n        {  {-3.43035, -6.5393, -16.786, -79.433},  // N [] = 1, 1%\n           {-2.86154, -2.8903, -4.234, -40.040},   // 5 %\n           {-2.56677, -1.5384, -2.809, 0}\n        },        // 10 %\n        {  {-3.89644, -10.9519, -33.527, 0},       // N [] = 2\n           {-3.33613, -6.1101, -6.823, 0},\n           {-3.04445, -4.2412, -2.720, 0}\n        },\n        {  {-4.29374, -14.4354, -33.195, 47.433},  // N [] = 3\n           {-3.74066, -8.5632, -10.852, 27.982},\n           {-3.45218, -6.2143, -3.718, 0}\n        },\n        {  {-4.64332, -18.1031, -37.972, 0},       // N [] = 4\n           {-4.09600, -11.2349, -11.175, 0},\n           {-3.81020, -8.3931, -4.137, 0}\n        },\n        {  {-4.95756, -21.8883, -45.142, 0},       // N [] = 5\n           {-4.41519, -14.0405, -12.575, 0},\n           {-4.13157, -10.7417, -3.784, 0}\n        },\n        {  {-5.24568, -25.6688, -57.737, 88.639},  // N [] = 6\n           {-4.70693, -16.9178, -17.492, 60.007},\n           {-4.42501, -13.1875, -5.104, 27.877}\n        },\n        {  {-5.51233, -29.5760, -69.398, 164.295},  // N [] = 7\n           {-4.97684, -19.9021, -22.045, 110.761},\n           {-4.69648, -15.7315, -5.104, 27.877}\n        },\n        {  {-5.76202, -33.5258, -82.189, 256.289},  // N [] = 8\n           {-5.22924, -23.0023, -24.646, 144.479},\n           {-4.95007, -18.3959, -7.344, 94.872}\n        },\n        {  {-5.99742, -37.6572, -87.365, 248.316},  // N [] = 9\n           {-5.46697, -26.2057, -26.627, 176.382},\n           {-5.18897, -21.1377, -9.484, 172.704}\n        },\n        {  {-6.22103, -41.7154, -102.680, 389.33},  // N [] = 10\n           {-5.69244, -29.4521, -30.994, 251.016},\n           {-5.41533, -24.0006, -7.514, 163.049}\n        },\n        {  {-6.43377, -46.0084, -106.809, 352.752},  // N [] = 11\n           {-5.90714, -32.8336, -30.275, 249.994},\n           {-5.63086, -26.9693, -4.083, 151.427}\n        },\n        {  {-6.63790, -50.2095, -124.156, 579.622},  // N [] = 12\n           {-6.11279, -36.2681, -32.505, 314.802},\n           {-5.83724, -29.9864, -2.686, 184.116}\n        }\n     };\n\n   matrix tau_ct_2010 [] =\n     {\n        {  {-3.95877, -9.0531, -28.428, -134.155},   // N [] = 1\n           {-3.41049, -4.3904, -9.036, -45.374},\n           {-3.12705, -2.5856, -3.925, -22.380}\n        },\n        {  {-4.32762, -15.4387, -35.679, 0},         // N [] = 2\n           {-3.78057, -9.5106, -12.074, 0},\n           {-3.49631, -7.0815, -7.538, 21.892}\n        },\n        {  {-4.66305, -18.7688, -49.793, 104.244},   // N [] = 3\n           {-4.11890, -11.8922, -19.031, 77.332},\n           {-3.83511, -9.0723, -8.504, 35.403}\n        },\n        {  {-4.96940, -22.4694, -52.599, 51.314},    // N [] = 4\n           {-4.42871, -14.5876, -18.228, 39.647},\n           {-4.14633, -11.2500, -9.873, 54.109}\n        },\n        {  {-5.25276, -26.2183, -59.631, 50.646},    // N [] = 5\n           {-4.71537, -17.3569, -22.660, 91.359},\n           {-4.43422, -13.6078, -10.238, 76.781}\n        },\n        {  {-5.51727, -29.9760, -75.222, 202.253},   // N [] = 6\n           {-4.98228, -20.3050, -25.224, 132.03},\n           {-4.70233, -16.1253, -9.836, 94.272}\n        },\n        {  {-5.76537, -33.9165, -84.312, 245.394},   // N [] = 7\n           {-5.23299, -23.3328, -28.955, 182.342},\n           {-4.95405, -18.7352, -10.168, 120.575}\n        },\n        {  {-6.00003, -37.8892, -96.428, 335.92},    // N [] = 8\n           {-5.46971, -26.4771, -31.034, 220.165},\n           {-5.19183, -21.4328, -10.726, 157.955}\n        },\n        {  {-6.22288, -41.9496, -109.881, 466.068},  // N [] = 9\n           {-5.69447, -29.7152, -33.784, 273.002},\n           {-5.41738, -24.2882, -8.584, 169.891}\n        },\n        {  {-6.43551, -46.1151, -120.814, 566.823},  // N [] = 10\n           {-5.90887, -33.0251, -37.208, 346.189},\n           {-5.63255, -27.2042, -6.792, 177.666}\n        },\n        {  {-6.63894, -50.4287, -128.997, 642.781},  // N [] = 11\n           {-6.11404, -36.4610, -36.246, 348.554},\n           {-5.83850, -30.1995, -5.163, 210.338}\n        },\n        {  {-6.83488, -54.7119, -139.800, 736.376},  // N [] = 12\n           {-6.31127, -39.9676, -37.021, 406.051},\n           {-6.03650, -33.2381, -6.606, 317.776}\n        }\n     };\n\n   matrix tau_ctt_2010 [] =\n     {\n        {  {-4.37113, -11.5882, -35.819, -334.047},  // N [] = 1\n           {-3.83239, -5.9057, -12.490, -118.284},\n           {-3.55326, -3.6596, -5.293, -63.559}\n        },\n        {  {-4.69276, -20.2284, -64.919, 88.884},    // N [] =2\n           {-4.15387, -13.3114, -28.402, 72.741},\n           {-3.87346, -10.4637, -17.408, 66.313}\n        },\n        {  {-4.99071, -23.5873, -76.924, 184.782},   // N [] = 3\n           {-4.45311, -15.7732, -32.316, 122.705},\n           {-4.17280, -12.4909, -17.912, 83.285}\n        },\n        {  {-5.26780, -27.2836, -78.971, 137.871},   // N [] = 4\n           {-4.73244, -18.4833, -31.875, 111.817},\n           {-4.45268, -14.7199, -17.969, 101.92}\n        },\n        {  {-5.52826, -30.9051, -92.490, 248.096},   // N [] = 5\n           {-4.99491, -21.2360, -37.685, 194.208},\n           {-4.71587, -17.0820, -18.631, 136.672}\n        },\n        {  {-5.77379, -34.7010, -105.937, 393.991},  // N [] = 6\n           {-5.24217, -24.2177, -39.153, 232.528},\n           {-4.96397, -19.6064, -18.858, 174.919}\n        },\n        {  {-6.00609, -38.7383, -108.605, 365.208},  // N [] = 7\n           {-5.47664, -27.3005, -39.498, 246.918},\n           {-5.19921, -22.2617, -17.910, 208.494}\n        },\n        {  {-6.22758, -42.7154, -119.622, 421.395},  // N [] = 8\n           {-5.69983, -30.4365, -44.300, 345.48},\n           {-5.42320, -24.9686, -19.688, 274.462}\n        },\n        {  {-6.43933, -46.7581, -136.691, 651.38},   // N [] = 9\n           {-5.91298, -33.7584, -42.686, 346.629},\n           {-5.63704, -27.8965, -13.880, 236.975}\n        },\n        {  {-6.64235, -50.9783, -145.462, 752.228},  // N [] = 10\n           {-6.11753, -37.056, -48.719, 473.905},\n           {-5.84215, -30.8119, -14.938, 316.006}\n        },\n        {  {-6.83743, -55.2861, -152.651, 792.577},  // N [] = 11\n           {-6.31396, -40.5507, -46.771, 487.185},\n           {-6.03921, -33.8950, -9.122, 285.164}\n        },\n        {  {-7.02582, -59.6037, -166.368, 989.879},  // N [] = 12\n           {-6.50353, -44.0797, -47.242, 543.889},\n           {-6.22941, -36.9673, -10.868, 418.414}\n        }\n     };\n\n   vector ret_vector = {0,0,0};\n\n   switch(trend)\n     {\n      case TREND_CONST_ONLY:\n         process(tau_c_2010,ret_vector,num_obs,nseries);\n         break;\n      case TREND_NONE:\n         process(tau_nc_2010,ret_vector,num_obs,nseries);\n         break;\n      case TREND_LINEAR_CONST:\n         process(tau_ct_2010,ret_vector,num_obs,nseries);\n         break;\n      case TREND_QUAD_LINEAR_CONST:\n         process(tau_ctt_2010,ret_vector,num_obs,nseries);\n         break;\n      default:\n         Print(\"Invalid input for trend argument\");\n         return ret_vector;\n     }\n\n   return ret_vector;\n  }\n//+------------------------------------------------------------------+\n//|helper function: evaluates a multiple variable polynomial         |\n//+------------------------------------------------------------------+\nvoid process(matrix &tau[], vector &out, ulong numobs, ulong ns)\n  {\n   if(numobs==ULONG_MAX)\n      out = tau[ns-1].Col(0);\n   else\n     {\n      for(ulong i=0; i<tau[ns-1].Cols(); i++)\n         out = out * (1.0/double(numobs)) + tau[ns-1].Col(tau[ns-1].Cols()-1-i);\n     }\n   return;\n  }\n\n  \n\n//+---------------------------------------------------------------------+\n//|Class CAdf                                                           |\n//|   encapsulates the the Augmented Dickey Fuller Test for Stationarity|\n//+---------------------------------------------------------------------+\n\nclass CAdf\n{\n private:\n  double m_adf_stat,  //adf statistic\n         m_bestic,    //optimal bic or aic\n         m_pvalue;    //p-value\n  ulong  m_usedlag;   //lag used for optimal reg model\n  vector m_critvals;  //estimated critical values\n  OLS    *m_ols;      //internal ordinary least squares reg model\n   // private methods\n bool   gridsearch(vector &LHS, matrix &RHS, ulong f_lag, ulong l_lag,ENUM_INFO_CRIT crit, double &b_ic, ulong &best_lag);\n bool   lagmat(matrix &in,matrix &out[],ulong mlag,ENUM_TRIM trim=TRIM_BOTH,ENUM_ORIGINAL original=ORIGINAL_IN);\n bool   prepare_lhs_rhs(vector &lhs, matrix &rhs, double &in[], double &in_diff[],ulong lag);\n \n \n public:\n  CAdf(void);\n  ~CAdf(void);\n  \n bool Adfuller(double &array[],ulong max_lag = 0,ENUM_TREND trend = TREND_CONST_ONLY, ENUM_INFO_CRIT autolag=INFO_AIC);\n vector CriticalValues(void) {  return m_critvals; }\n double AdfStatistic(void)   {  return m_adf_stat; }\n double Pvalue(void)         {  return m_pvalue;   }\n};\n//+------------------------------------------------------------------+\n//| Constructor                                                      |\n//+------------------------------------------------------------------+\nCAdf::CAdf(void)\n{\n m_ols = new OLS();\n \n m_adf_stat = m_bestic = m_pvalue = EMPTY_VALUE;\n \n m_usedlag = 0;\n \n m_critvals = vector::Zeros(3);\n}\n//+------------------------------------------------------------------+\n//| Destructor                                                       |\n//+------------------------------------------------------------------+\nCAdf::~CAdf(void)\n{\n if(CheckPointer(m_ols)==POINTER_DYNAMIC)\n    delete m_ols;\n\n}\n//+------------------------------------------------------------------+\n//| Augmented Dickey-Fuller unit root test                           |\n//+------------------------------------------------------------------+\nbool CAdf::Adfuller(double&array[],ulong max_lag = 0,ENUM_TREND trend = TREND_CONST_ONLY, ENUM_INFO_CRIT autolag=INFO_AIC)\n  {\n//---  \n   if(CheckPointer(m_ols)==POINTER_INVALID)\n    {\n     Print(\"Critical internal error: Invalid Pointer to OLS object\");\n     return false;\n    }\n//---\n   if(array[ArrayMaximum(array)]==array[ArrayMinimum(array)])\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Invalid input series is made of constants \");\n      return false;\n     }\n//---\n   uint nobs = array.Size();\n//---\n   uint ntrend = (trend>1)?uint(trend)-1:uint(trend);\n//---\n   ulong maxlag;\n//---\n   if(!max_lag)\n     {\n      maxlag =  uint(ceil(12.0 * pow(nobs / 100.0, 1.0 / 4.0)));\n      maxlag =  MathMin(nobs / 2 - ntrend - 1, maxlag);\n\n      if(ntrend>=2)\n        {\n         Print(\"sample size is too short to use selected regression component: Adjust trend input value \");\n         return false;\n        }\n     }\n   else\n     {\n      maxlag = max_lag;\n\n      if(ntrend>=2)\n        {\n         Print(\"sample size is too short to use selected regression component: Adjust trend input value \");\n         return false;\n        }\n\n      if(maxlag>floor(nobs/2-ntrend-1))\n        {\n         Print(\"maxlag must be less than (nobs/2 - 1 - ntrend) \\n where ntrend is the number of included deterministic regressors\");\n         return false;\n        }\n     }\n//---\n   double diff[];\n//---\n   if(!MathDifference(array,1,diff))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Error differencing error : \",::GetLastError());\n      return false;\n     }\n//---\n   vector xdshort, tmp;\n//---\n   matrix xdiff;\n//---\n   if(!xdiff.Resize(diff.Size(),1) || !tmp.Resize(diff.Size()) || !tmp.Assign(diff) || !xdiff.Col(tmp,0))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize or assign error : \",::GetLastError());\n      return false;\n     }\n//---\n   matrix xdall[];\n//---\n   if(!lagmat(xdiff,xdall,maxlag))\n      return false;\n//---\n   if(!prepare_lhs_rhs(xdshort,xdall[0],array,diff,maxlag))\n      return false;\n\n//---\n   matrix fullRHS;\n   double bestic;\n   ulong startlag,bestlag,usedlag;\n//---\n   if(autolag!=INFO_NONE)\n     {\n      if(trend!=TREND_NONE)\n        {\n         if(!addtrend(xdall[0],fullRHS,trend,true))\n            return false;\n        }\n      else\n        {\n         if(!fullRHS.Copy(xdall[0]))\n           {\n            Print(__FUNCTION__,\" \",__LINE__,\" Resize or assign error : \",::GetLastError());\n            return false;\n           }\n        }\n\n      startlag = fullRHS.Cols() - xdall[0].Cols() + 1;\n\n      gridsearch(xdshort,fullRHS,startlag,maxlag,autolag,bestic,bestlag);\n\n      bestlag-=startlag;\n\n      if(!lagmat(xdiff,xdall,bestlag))\n         return false;\n\n      if(!prepare_lhs_rhs(xdshort,xdall[0],array,diff,bestlag))\n         return false;\n\n      usedlag = bestlag;\n\n     }\n   else\n     {\n      usedlag = maxlag;\n      bestic = 0;\n     }\n//---\n   m_usedlag = usedlag;\n   m_bestic = bestic;\n//---   \n   matrix xv;\n//---\n   xv.Resize(xdall[0].Rows(),ulong(usedlag+1),2);\n//---\n   for(ulong k=0; k<ulong(usedlag+1); k++)\n     {\n      if(!xv.Col(xdall[0].Col(k),k))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Error adding column: \",::GetLastError());\n         return false;\n        }\n     }\n//---\n   if(trend!=TREND_NONE)\n     {\n      if(!addtrend(xv,fullRHS,trend))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Error processing input \");\n         return false;\n        }\n     }\n   else\n     {\n      if(!fullRHS.Copy(xv))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Error processing input \");\n         return false;\n        }\n     }\n\n\n   if(!m_ols.Fit(fullRHS, xdshort))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Error OLS fit: \",::GetLastError());\n      return false;\n     }\n//---\n   vector adfstat = m_ols.Tvalues();\n//---\n   m_adf_stat = adfstat[0];\n//---\n   m_pvalue = mackinnonp(m_adf_stat,trend);\n//---\n   m_critvals = mackinnoncrit(1,trend,fullRHS.Rows());\n//---\n   return true;\n  }\n//+------------------------------------------------------------------+\n//| helper function prepares rhs of equation                         |\n//+------------------------------------------------------------------+\nbool CAdf::prepare_lhs_rhs(vector &lhs, matrix &rhs, double &in[], double &in_diff[],ulong lag)\n  {\n   ulong len= rhs.Rows();\n\n   if(!lhs.Resize(len))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize or assign error : \",::GetLastError());\n      return false;\n     }\n\n   vector cpy;\n   if(!cpy.Resize(len))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize or assign error : \",::GetLastError());\n      return false;\n     }\n\n   ulong mm = ulong(in.Size())-(len+1);\n\n   for(ulong i=mm; i<ulong(in.Size()-1); i++)\n      cpy[i-mm] = in[i];\n\n   if(!rhs.Col(cpy,0))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize or assign error : \",::GetLastError());\n      return false;\n     }\n\n   mm = ulong(in_diff.Size())-(len);\n\n   for(ulong i=mm; i<ulong(in_diff.Size()); i++)\n      lhs[i-mm] = in_diff[i];\n\n   return true;\n  }\n//+-----------------------------------------------------------------------------+\n//|  helper function performs search for optimal lag based in selected criterion|\n//+-----------------------------------------------------------------------------+\nbool CAdf::gridsearch(vector &LHS, matrix &RHS, ulong f_lag, ulong l_lag,ENUM_INFO_CRIT crit, double &b_ic, ulong &best_lag)\n  {\n//---\n   matrix xv;\n//---\n   b_ic=DBL_MAX;\n//---\n   double ic=0;\n//---\n   for(ulong i = f_lag; i<f_lag+l_lag+1; i++)\n     {\n      //---\n      xv.Resize(RHS.Rows(),ulong(i),2);\n      //---\n      //vector v;\n      //---\n      for(ulong k=0; k<ulong(i); k++)\n        {\n         if(!xv.Col(RHS.Col(k),k))\n           {\n            Print(__FUNCTION__,\" \",__LINE__,\" Error adding column: \",::GetLastError());\n            return false;\n           }\n        }\n\n      //---\n      if(!m_ols.Fit(xv, LHS))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Error OLS fit: \",::GetLastError());\n         return false;\n        }\n      //---\n      switch(crit)\n        {\n         case INFO_AIC:\n            ic = m_ols.Aic();\n            break;\n         case INFO_BIC:\n            ic = m_ols.Bic();\n            break;\n        }\n      //---\n      if(ic<b_ic)\n        {\n         b_ic = ic;\n         best_lag = i;\n        }\n      //---\n     }\n//---\n\n   return true;\n  }\n//+------------------------------------------------------------------+\n//| helper function: transforms rhs matrix                           |\n//+------------------------------------------------------------------+\nbool CAdf::lagmat(matrix &in,matrix &out[],ulong mlag,ENUM_TRIM trim=TRIM_BOTH,ENUM_ORIGINAL original=ORIGINAL_IN)\n  {\n//---\n   ulong nobs=in.Rows();\n   ulong nvars =in.Cols();\n   ulong dropidx = 0;\n//---\n   if(mlag>=nobs)\n     {\n      Print(\"mlag should be < number of rows of the input matrix\");\n      return false;\n     }\n//---\n   if(original==ORIGINAL_EX || original==ORIGINAL_SEP)\n      dropidx = nvars;\n//---\n   matrix nn;\n//---\n   if(!nn.Resize(nobs + mlag,nvars * (mlag + 1)))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n      return false;\n     }\n//---\n   nn.Fill(0.0);\n//---\n   ulong maxlag=mlag;\n//---\n   ulong row_end,row_start,col_end,col_start,j,z;\n   row_end=row_start=col_end=col_start=j=z=0;\n//---\n   for(ulong k = 0; k<(maxlag+1); k++)\n     {\n      row_start = maxlag-k;\n      row_end = nobs+maxlag-k;\n      col_start = nvars*(maxlag-k);\n      col_end = nvars*(maxlag-k+1);\n      j = 0;\n      for(ulong irow = row_start; irow < row_end; irow++, j++)\n        {\n         z = 0;\n         for(ulong icol = col_start; icol < col_end; icol++, z++)\n            nn[irow][icol]=in[j][z];\n        }\n     }\n//---\n   ulong startobs,stopobs;\n   if(trim==TRIM_NONE || trim==TRIM_FORWARD)\n      startobs=0;\n   else\n      startobs=maxlag;\n//---\n   if(trim==TRIM_NONE || trim==TRIM_BACKWARD)\n      stopobs=nn.Rows();\n   else\n      stopobs=nobs;\n//---\n   if(dropidx)\n      ArrayResize(out,2);\n   else\n      ArrayResize(out,1);\n//---\n   if(!out[0].Resize(stopobs-startobs,nn.Cols()-dropidx))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n      return false;\n     }\n//---\n   for(ulong irow = startobs; irow<stopobs; irow++)\n      for(ulong icol=dropidx; icol<nn.Cols(); icol++)\n         out[0][irow-startobs][icol-dropidx]=nn[irow][icol];\n//---\n   if(out.Size()>1)\n     {\n      if(!out[1].Resize(stopobs-startobs,dropidx))\n        {\n         Print(__FUNCTION__,\" \",__LINE__,\" Resize Error \", ::GetLastError());\n         return false;\n        }\n      //---\n      for(ulong irow = startobs; irow<stopobs; irow++)\n         for(ulong icol=0; icol<out[1].Cols(); icol++)\n            out[1][irow-startobs][icol]=nn[irow][icol];\n     }\n//---\n   return true;\n  }\n"
  },
  {
    "path": "Stats Models/ARIMA.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                        ARIMA.mqh |\n//|                                     Copyright 2023, Omega Joctan |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n//+------------------------------------------------------------------+\n//| defines                                                          |\n//+------------------------------------------------------------------+\n\n#include <MALE5\\Linear Models\\Linear Regression.mqh>\n#include <MALE5\\MatrixExtend.mqh>\n\nstruct ar_struct\n {\n   vector residuals; //The differences between actual and predicted values.\n   vector theta; //The coefficients of the AR model.\n   double intercept; //The intercept term from the regression model.\n };\n\nstruct ma_struct\n {\n   vector theta;\n   double intercept;\n };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n \nclass CARIMA\n  {\nprotected:\n\n   CLinearRegression  *ar_lr, *ma_lr;\n   \n   \n   vector Shift(const vector &v, int shift); \n   vector Pad(const vector &v, int padSize, double padValue=EMPTY_VALUE);\n   \n   ar_struct AR(const uint p, const vector &series_data);\n   ma_struct MA(const uint q, const vector &residuals);\n   \n   uint __p__,__d__, __q__;  \n    \n   ar_struct ar_parameters;\n   ma_struct ma_parameters;\n                     \npublic:\n                     CARIMA(const uint p, const uint d, const uint q);\n                    ~CARIMA(void);\n                    \n                     vector difference(const vector &ts, uint interval=1);\n                     double inverse_difference(const vector &history, double y_hat, uint interval=1);\n                    \n                     void fit(const vector &series);\n                     vector predict(const vector &series, const uint steps=10);\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCARIMA::CARIMA(const uint p, const uint d, const uint q)\n :__p__(p),\n  __q__(q),\n  __d__(d)\n {\n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCARIMA::~CARIMA(void)\n {\n   if (CheckPointer(ar_lr) != POINTER_INVALID)\n     delete ar_lr;\n     \n   if (CheckPointer(ma_lr) != POINTER_INVALID)\n     delete ma_lr;\n }\n//+------------------------------------------------------------------+\n//|   This function shifts series_data similarly to                  |\n//|   pandas.DataFrame.shift                                         |\n//+------------------------------------------------------------------+\nvector CARIMA::Shift(const vector &v, int shift) \n {\n   int size = (int)v.Size();\n   \n   vector new_v(v.Size());\n   new_v.Fill(EMPTY_VALUE);\n   \n   // If shift is positive, shift right\n   if(shift > 0) \n    {\n      for(int i = size - 1; i >= shift; i--) \n        new_v[i] = v[i - shift];\n     }\n   \n   // If shift is negative, shift left\n   else if(shift < 0) \n    {\n      shift = -shift;\n      for(int i = 0; i < size - shift; i++) \n        new_v[i] = v[i + shift];\n     }\n     \n   return new_v;\n }\n\n//+------------------------------------------------------------------+\n//|      Function to pad a vector with a specified value             |\n//+------------------------------------------------------------------+\nvector CARIMA::Pad(const vector &v, int padSize, double padValue=EMPTY_VALUE)\n{\n   int originalSize = (int)v.Size();\n   int newSize = originalSize + padSize;\n   \n   vector results(newSize); //a vector to hold the padded results\n   \n   // Fill the beginning of the array with the padValue\n   for (int i = 0; i < padSize; i++)\n      results[i] = padValue;\n   \n   // Copy the original array to the new array after the padding\n   for (int i = 0; i < originalSize; i++)\n      results[i + padSize] = v[i];\n   \n   return results;\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|      Perform differencing to make time series stationary         |\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CARIMA::difference(const vector &ts, uint interval=1)\n {\n   if (interval>=ts.Size())\n     {\n       printf(\"%s fatal, interval=%d must be less than the timeseries vector size=%d\",__FUNCTION__,interval,ts.Size());\n       vector empty={};\n       return empty;\n     }\n   \n   vector diff(ts.Size()-interval);\n   \n   for (uint i=interval, count=0; i<ts.Size(); i++)\n     diff[i-interval] = ts[i] - ts[i-interval];\n     \n   return diff;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|  To invert the differencing, we need to add the differenced value|\n//|  y_hat back to the last observed value before the differencing.  |\n//|                                                                  |\n//|  Parameters                                                      |     \n//|  history: The original time series data before differencing.     |\n//|  yhat: The differenced value or forecast that we want to convert |\n//|         back to the original scale.                              |\n//|  interval: The differencing interval, default value is 1 for     |\n//|            first-order differencing                              |\n//|                                                                  |\n//+------------------------------------------------------------------+\ndouble CARIMA::inverse_difference(const vector &history, double y_hat, uint interval=1)\n {\n   return y_hat + history[history.Size()-interval];\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|  This function fits an AutoRegressive model of order p to the    |\n//|  time series data by leveraging linear regression.               |\n//|                                                                  |\n//|  It predicts the current value of the time series based on its   |\n//|  previous p values, calculates the prediction errors, and        |\n//|  provides the arima model coefficients and intercept values.     |\n//|                                                                  |   \n//|                                                                  |\n//|  p: The order of the AR model, which specifies how many lagged   |\n//|     values of the time series to include as predictors.          |\n//|  series_data: The time series series_data.                       |\n//|                                                                  |\n//+------------------------------------------------------------------+\nar_struct CARIMA::AR(const uint p, const vector &series_data)\n {\n   ar_struct ret_struct;\n   \n   ulong size = series_data.Size();\n   \n   matrix autoregressive_series_data(size, p+1);\n   autoregressive_series_data.Col(series_data, 0);\n   \n   vector shifted_series_data = {};\n \n    for (ulong i=1; i<p+1; i++) //generate lagged values\n      {\n         shifted_series_data = Shift(series_data, (uint)i);\n         autoregressive_series_data.Col(shifted_series_data, i);\n      } \n      \n//---\n      \n   autoregressive_series_data = MatrixExtend::Slice(autoregressive_series_data,p,-1); \n   \n//--- Since the y vector is the first column of this matrix\n   \n   matrix X;\n   vector y;\n   \n   MatrixExtend::XandYSplitMatrices(autoregressive_series_data, X, y, 0); //index of 0 gets the first column assigned as y vector\n   \n//--- Fitting a linear regression model to the outo-regressive series_data\n   \n   ar_lr = new CLinearRegression();\n   \n   ar_lr.fit(X, y);\n   vector y_pred = ar_lr.predict(X);\n      \n//---\n\n   ret_struct.residuals = y - y_pred;\n   ret_struct.theta = ar_lr.coeff_;\n   ret_struct.intercept = ar_lr.intercept_;\n   \n   if (MQLInfoInteger(MQL_DEBUG))\n     printf(\"AR(p=%d) model - RMSE: %.4f\",p,y_pred.RegressionMetric(y, REGRESSION_RMSE));\n   \n   return ret_struct;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|  The MA function constructs a Moving Average (MA) model by       |\n//|  regressing the current residuals against the past q residuals.  |\n//|  It uses linear regression to determine the coefficients theta   |\n//|  and the intercept that best fit the relationship between the    |\n//|  residuals and their lagged values.                              |   \n//|  The resulting model helps in forecasting future residuals based |   \n//|  on past errors.                                                 |\n//|                                                                  |\n//|  Parameters:                                                     |\n//|  q: The order of the MA model, indicating how many lagged        |\n//|     residuals (errors) should be considered.                     |\n//|  residuals: The array of residuals (errors) from the previous AR |\n//|     model fitting.                                               |\n//|                                                                  |\n//+------------------------------------------------------------------+\nma_struct CARIMA::MA(const uint q,const vector &residuals)\n {\n   ma_struct ret_struct;\n   \n   ulong size = residuals.Size();\n   \n   matrix autoregressive_residuals(size, q+1);\n   autoregressive_residuals.Col(residuals, 0);\n   \n   vector shifted_eesiduals = {};\n \n    for (ulong i=1; i<q+1; i++) //This loop creates lagged versions of the residuals.\n      {\n         shifted_eesiduals = Shift(residuals, (uint)i);\n         autoregressive_residuals.Col(shifted_eesiduals, i);\n      } \n     \n//---\n      \n   autoregressive_residuals = MatrixExtend::Slice(autoregressive_residuals,q,-1); \n   \n//--- Since the y vector is the first column of this matrix\n   \n   matrix X;\n   vector y;\n   \n   MatrixExtend::XandYSplitMatrices(autoregressive_residuals, X, y, 0); //index of 0 gets the first column assigned as y vector\n   \n//--- Fitting a linear regression model to the outo-regressive residuals\n   \n   ma_lr = new CLinearRegression();\n   ma_lr.fit(X, y);\n   vector y_pred = ma_lr.predict(X);\n      \n//---\n   \n   ret_struct.theta = ma_lr.coeff_;\n   ret_struct.intercept = ma_lr.intercept_;   \n\n//---\n\n   if (MQLInfoInteger(MQL_DEBUG))\n     printf(\"MA(q=%d) model - RMSE: %.4f\",q,y_pred.RegressionMetric(y, REGRESSION_RMSE));\n     \n   return ret_struct;   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CARIMA::fit(const vector &series)\n {\n   ar_parameters = AR(__p__, series);\n   ma_parameters = MA(__q__, ar_parameters.residuals); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CARIMA::predict(const vector &series,const uint steps=10)\n {\n    vector forecasted_values(steps);\n    vector temp_series = series;\n    \n    //--- We initialize the residuals for the new data\n    \n    vector temp_residuals(series.Size() + steps);\n    temp_residuals.Fill(0);\n    \n    for (uint step=0; step<steps; step++)\n      {\n        \n        //--- Auto-regressive part\n        \n           vector ar_terms = MatrixExtend::Slice(temp_series,temp_series.Size()-__p__, -1);\n           MatrixExtend::Reverse(ar_terms);\n           \n           double ar_part = ar_parameters.theta.MatMul(ar_terms) + ar_parameters.intercept;\n           \n        //--- Moving-average part | Generating MA terms\n        \n           vector ma_terms = MatrixExtend::Slice(temp_residuals,temp_residuals.Size()-__q__, -1);\n           MatrixExtend::Reverse(ma_terms);\n           \n           double ma_part = ma_parameters.theta.MatMul(ma_terms) + ma_parameters.intercept;\n           \n        //--- Calculate forecast value difference \n        \n           double forecast_value_diff = ar_part + ma_part;\n           double new_value = temp_series[temp_series.Size()-1] + forecast_value_diff; // We use the last value in the series to convert the differenced forecast back to original scale\n\n           forecasted_values[step] = new_value;\n           \n         //--- Update the temp_residuals with the new forecasted value\n           \n           temp_series = MatrixExtend::concatenate(temp_series, new_value);\n           \n           Print(\"Temp series\\n\",temp_series);\n           \n           double new_residual = forecast_value_diff;\n           temp_residuals = MatrixExtend::concatenate(temp_residuals, new_residual);\n           \n           Print(\"Temp residuals\\n\",temp_residuals);\n      }\n          \n    return forecasted_values;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Stats Models/OLS.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                          OLS.mqh |\n//|                                  Copyright 2023, MetaQuotes Ltd. |\n//|                                             https://www.mql5.com |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2023, MetaQuotes Ltd.\"\n#property link      \"https://www.mql5.com\"\n//+------------------------------------------------------------------+\n//| Ordinary least squares class                                     |\n//+------------------------------------------------------------------+\nclass OLS\n  {\nprivate:\n   bool   m_const_prepended;    //check column for constant in design matrix\n   matrix m_exog,               //design matrix\n          m_pinv,               //pseudo-inverse of matrix\n          m_cov_params,         //covariance of matrix\n          m_m_error,            //error matrix\n          m_norm_cov_params;    //normalized covariance matrix\n   vector m_endog,              //dependent variables\n          m_weights,            //weights\n          m_singularvalues,     //singular values of solution\n          m_params,             //coefficients of regression model(solution)\n          m_tvalues,            //test statistics of model\n          m_bse,                //standard errors of model\n          m_const_cols,         //mark constant columns in design matrix\n          m_resid;              //residuals of model\n   ulong  m_obs,                //number of observations\n          m_model_dof,          //degrees of freedom of model\n          m_resid_dof,          //degrees of freedom of residuals\n          m_kconstant,          //number of constants\n          m_rank;               //rank of design matrix\n   double m_aic,                //Akiake information criteria\n          m_bic,                //Bayesian information criteria\n          m_scale,              //scale of model\n          m_llf,                //loglikelihood of model\n          m_sse,                //sum of squared errors\n          m_rsqe,               //r-squared of model\n          m_centeredtss,        //centered sum of squares\n          m_uncenteredtss;      //uncentered sum of squares\n   uint              m_error;              //error flag\n   // private methods\n   ulong             countconstants(void);\n   void              scale(void);\n   void              sse(void);\n   void              rsqe(void);\n   void              centeredtss(void);\n   void              uncenteredtss(void);\n   void              aic(void);\n   void              bic(void);\n   void              bse(void);\n   void              llf(void);\n   void              tvalues(void);\n   void              covariance_matrix(void);\n\n\npublic:\n   //constructor\n                     OLS(void);\n   //destructor\n                    ~OLS(void);\n   //public methods\n   bool              Fit(const matrix & x, const vector &y);\n   double            predict(vector &x);\n   double            predict(double x);\n   //get properties of OLS model\n   ulong             ModelDOF(void) { if(m_error) return 0; else return m_model_dof;}\n   ulong             ResidDOF(void) { if(m_error) return 0; else return m_resid_dof;}\n   double            Scale(void)  { if(m_error) return EMPTY_VALUE; else return m_scale;    }\n   double            Aic(void)    { if(m_error) return EMPTY_VALUE; else return m_aic;      }\n   double            Bic(void)    { if(m_error) return EMPTY_VALUE; else return m_bic;    }\n   double            Sse(void)    { if(m_error) return EMPTY_VALUE; else return m_sse;    }\n   double            Rsqe(void)   { if(m_error) return EMPTY_VALUE; else return m_rsqe;   }\n   double            C_tss(void)  { if(m_error) return EMPTY_VALUE; else return m_centeredtss;}\n   double            Loglikelihood(void) { if(m_error) return EMPTY_VALUE; return m_llf; }\n   vector            Tvalues(void) { if(m_error) return m_m_error.Col(0); return m_tvalues; }\n   vector            Residuals(void) { if(m_error) return m_m_error.Col(0); return m_resid; }\n   vector            ModelParameters(void) { if(m_error) return m_m_error.Col(0); return m_params; }\n   vector            Bse(void) { if(m_error) return m_m_error.Col(0);  return m_bse; } \n   matrix            CovarianceMatrix(void) { if(m_error) return m_m_error; return m_cov_params; }\n  };\n//+------------------------------------------------------------------+\n//|  constructor                                                     |\n//+------------------------------------------------------------------+\nOLS::OLS(void)\n  {\n   m_kconstant = 0;\n   m_obs = 0;\n   m_model_dof = 0;\n   m_resid_dof = 0;\n   m_kconstant = 0;\n   m_rank = 0;\n   m_aic = 0;\n   m_bic = 0;\n   m_scale = 0;\n   m_llf   = 0;\n   m_error = 1;\n\n   m_m_error.Resize(2,2);\n   m_m_error.Fill(EMPTY_VALUE);\n\n  }\n//+------------------------------------------------------------------+\n//|Destructor                                                        |\n//+------------------------------------------------------------------+\nOLS::~OLS(void)\n  {\n\n  }\n//+------------------------------------------------------------------+\n//| Fit data                                                         |\n//+------------------------------------------------------------------+\nbool OLS::Fit(const matrix & x, const vector &y)\n  {\n//---re-initialize internal properties\n   m_endog = y;\n//---\n   m_exog  = x;\n//---\n   m_kconstant = countconstants();\n//---\n   m_error = 0;\n//---\n   m_model_dof = m_resid_dof = m_rank = m_obs= 0;\n//---\n   m_aic = m_bic= m_llf=0;\n//---\n   m_obs=m_exog.Rows();\n//---\n   m_weights.Resize(m_obs);\n//---\n   m_weights.Fill(1.0);\n//---\n   m_rank = m_exog.Rank();\n//---\n   m_model_dof = m_rank - m_kconstant;\n//---\n   m_resid_dof = m_obs - m_rank;\n//---\n   if(y.Size()!=x.Rows())\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Error Invalid x Rows in x not equal to length of y\");\n      m_error = 1;\n      return false;\n     }\n//---\n   matrix U,V;\n//---\n   ::ResetLastError();\n//--- SVD operation\n   if(!m_exog.SVD(U,V,m_singularvalues))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" SVD operation failed : \",::GetLastError());\n      m_error = 1;\n      return false;\n     }\n//--- Compute the pseudo-inverse of a matrix by the Moore-Penrose method\n   m_pinv = m_exog.PInv();\n//--- transpose m_pinv\n   matrix pinv_t = m_pinv.Transpose();\n//--- normalize covariance matrix\n   m_norm_cov_params = m_pinv.MatMul(pinv_t);\n//---\n   matrix diag;\n   if(!diag.Diag(m_singularvalues))\n     {\n      Print(__FUNCTION__,\" \",__LINE__,\" Diag operation failed : \",::GetLastError());\n      m_error = 1;\n      return false;\n     }\n//--- rank of diag\n   m_rank = diag.Rank();\n//--- estimate parameters of the model\n   m_params = m_pinv.MatMul(m_endog);\n//--- caluclate its degrees of freedom\n   m_model_dof = m_rank - m_kconstant;\n//--- as well as those of the residuals\n   m_resid_dof = m_obs - m_rank;\n//---get the residuals\n   m_resid = m_endog - m_exog.MatMul(m_params);\n//---compute scale of model\n   scale();\n//--- compute covariance matrix\n   covariance_matrix();\n//--- compute standard errors\n   bse();\n//--- compute sum of squared errors\n   sse();\n//--- compute loglikelihood function\n   llf();\n//--- compute centered sum of squares\n   centeredtss();\n//--- compute uncentered sum of squares\n   uncenteredtss();\n//--- compute r-squared\n   rsqe();\n//--- compute AIC\n   aic();\n//--- compute BIC\n   bic();\n//--- compute tvalues\n   tvalues();\n//---\n   return true;\n  }\n//+------------------------------------------------------------------+\n//| predict next value based on model parameters                     |\n//+------------------------------------------------------------------+\ndouble OLS::predict(vector &x)\n  {\n//---\n   if(m_error)\n     {\n      Print(\"Invalid model\");\n      return EMPTY_VALUE;\n     }\n//---\n   if(x.Size()!=(m_params.Size()-m_kconstant))\n     {\n      Print(\"invalid x: supplied vector does not match size of model parameters\");\n      return EMPTY_VALUE;\n     }\n//---\n   double prediction = 0;\n//---   \n   for(ulong i = 0,k = 0;i<m_const_cols.Size(); i++)\n      {\n       if(!m_const_cols[i])\n          prediction+=(m_params[i]*x[k++]); \n       else\n          prediction+=(m_params[i]);   \n      }           \n//---\n   return prediction;\n  }\n//+------------------------------------------------------------------+\n//| predict next value based on model parameters                     |\n//+------------------------------------------------------------------+\ndouble OLS::predict(double x)\n  {\n//---\n   if(m_error || (m_params.Size()-m_kconstant)>1)\n     {\n      if(m_error)\n         Print(\"Invalid model\");\n      else\n         Print(\"invalid x: insufficient number of predictors supplied\");   \n      return EMPTY_VALUE;\n     }\n//---\n  double prediction = 0;\n//---\n  if(m_kconstant)\n      prediction=(m_const_cols[0])?m_params[0]+(m_params[1]*x):m_params[1]+(m_params[0]*x);\n  else\n      prediction=m_params[0]*x;    \n//---\n   return prediction;\n  }\n//+------------------------------------------------------------------+\n//|Count the number of constants in RHS matrix                       |\n//+------------------------------------------------------------------+\nulong OLS::countconstants(void)\n  {\n   ulong count = 0;\n   vector temp;\n   \n   m_const_cols.Resize(m_exog.Cols());\n   m_const_cols.Fill(0.0);\n   \n   for(ulong i =0; i<m_exog.Cols(); i++)\n     {\n      temp = m_exog.Col(i);\n      if((temp.Max()-temp.Min()) == 0.0)\n         {\n          count++;\n          m_const_cols[i]=1.0;\n         }\n\n     }\n     \n   return count;\n  }\n//+------------------------------------------------------------------+\n//|scale factor for the covariance matrix                            |\n//+------------------------------------------------------------------+\nvoid OLS::scale(void)\n  {\n   m_scale =  m_resid.Dot(m_resid)/double(m_resid_dof);\n  }\n//+------------------------------------------------------------------+\n//| the variance/covariance matrix                                   |\n//+------------------------------------------------------------------+\nvoid OLS::covariance_matrix(void)\n  {\n//---\n   m_cov_params=m_norm_cov_params*m_scale;\n  }\n//+------------------------------------------------------------------+\n//| standard errors of the parameter estimates                       |\n//+------------------------------------------------------------------+\nvoid OLS::bse(void)\n  {\n   m_bse=m_cov_params.Diag();\n   m_bse=MathSqrt(m_bse);\n  }\n//+------------------------------------------------------------------+\n//| sum of squared errors                                            |\n//+------------------------------------------------------------------+\nvoid OLS::sse(void)\n  {\n   vector sqresid=MathPow(m_resid,2);\n\n   m_sse = sqresid.Sum();\n\n  }\n//+------------------------------------------------------------------+\n//|likelihood function for the OLS model                             |\n//+------------------------------------------------------------------+\nvoid OLS::llf(void)\n  {\n   double obs2=double(m_obs)/2.0;\n   m_llf = -obs2*log(2.0*M_PI) - obs2*log(m_sse / double(m_obs)) - obs2;\n//m_llf = -1*obs2*MathLog(2.0*M_PI*m_scale) - m_sse/(2.0*m_scale);\n  }\n//+------------------------------------------------------------------+\n//| Akaike's information criteria                                    |\n//+------------------------------------------------------------------+\nvoid OLS::aic(void)\n  {\n   double params = double(m_model_dof)+double(m_kconstant);\n\n   m_aic = -2*m_llf+2*params;\n\n  }\n//+------------------------------------------------------------------+\n//|Bayes' information criteria                                       |\n//+------------------------------------------------------------------+\nvoid OLS::bic(void)\n  {\n   double params = double(m_model_dof)+double(m_kconstant);\n\n   m_bic = -2*m_llf+MathLog(m_obs)*params;\n\n  }\n//+------------------------------------------------------------------+\n//|t-statistic for a given parameter estimate                        |\n//+------------------------------------------------------------------+\nvoid OLS::tvalues(void)\n  {\n   m_tvalues = m_params/m_bse;\n  }\n//+------------------------------------------------------------------+\n//|total  sum of squares centered about the mean                     |\n//+------------------------------------------------------------------+\nvoid OLS::centeredtss(void)\n  {\n   vector centered_endog = m_endog - m_endog.Mean();\n//---\n   m_centeredtss = centered_endog.Dot(centered_endog);\n  }\n//+------------------------------------------------------------------+\n//| The sum of the squared values of the endogenous response variable|\n//+------------------------------------------------------------------+\nvoid OLS::uncenteredtss(void)\n  {\n   m_uncenteredtss = m_endog.Dot(m_endog);\n  }\n//+------------------------------------------------------------------+\n//|R-squared of the model                                            |\n//+------------------------------------------------------------------+\nvoid OLS::rsqe(void)\n  {\n   m_rsqe = (m_kconstant)? 1 - m_sse/m_centeredtss: 1 - m_sse/m_uncenteredtss;\n  }\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "Tensors.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                      Tensors.mqh |\n//|                                    Copyright 2022, Fxalgebra.com |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Fxalgebra.com\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CMatrix\n  {\n   public:\n         matrix Matrix;\n  };\n  \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass C3DTensor\n  {\nCMatrix* matrices[]; \n\npublic:\n                     C3DTensor(void); //For one dimension tensor\n                    ~C3DTensor(void);\n                    \n                    bool   Init(uint size);\n                    bool   Append(matrix<double> &__matrix__);\n                    \n                    CMatrix *GetObj(int index);\n                    //virtual matrix operator[](const int index) { return Get(index); }\n                    CMatrix* operator[](const int index) { return GetObj(index); }\n                    void   Print_();\n                    \n                    void   Delete();\n                    uint   Size(); //returns tensor's size\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nC3DTensor::C3DTensor(void)\n {   \n \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nC3DTensor::~C3DTensor(void)\n {\n   for (uint i=0; i<matrices.Size(); i++)\n     if (CheckPointer(matrices[i]) != POINTER_INVALID)\n       delete matrices[i];\n\n   ArrayFree(matrices);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|  This function initilalizes the 3D tensor by creating empty      |\n//|  matrices to the tensor memory                                   |\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool C3DTensor::Init(uint size)\n {\n   if (size==0)\n     return false;\n     \n   ArrayResize(this.matrices, size);\n   return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint C3DTensor::Size()\n {\n   return this.matrices.Size();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool C3DTensor::Append(matrix<double> &__matrix__)\n {\n   if (ArrayResize(matrices, matrices.Size()+1)<0)\n    return false;\n    \n   uint SIZE = matrices.Size();\n   matrices[SIZE-1] = new CMatrix();\n   matrices[SIZE-1].Matrix = __matrix__; //Add the new matrix to the newly created tensor index\n   \n   return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid C3DTensor::Print_(void)\n {\n   for (uint i=0; i<matrices.Size(); i++)\n     Print(\"TENSOR INDEX [\",i,\"] matrix-size=(\",this.matrices[i].Matrix.Rows(),\"x\",this.matrices[i].Matrix.Cols(),\")\\n\",this.matrices[i].Matrix); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCMatrix *C3DTensor::GetObj(int index)\n {\n   if (index<-1 || index > int(matrices.Size()))\n    {\n      printf(\"%s failed, index out of range. Line %d\",__FUNCTION__, __LINE__);\n      return this.matrices[index==-1?matrices.Size()-1: index];\n    }\n   \n   return this.matrices[index==-1?matrices.Size()-1: index]; //if the selected position is -1 we obtain the last matrix in our tensor\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid C3DTensor::Delete(void)\n {\n   for (ulong i=0; i<matrices.Size(); i++)\n    {\n      this.matrices[i].Matrix.Resize(0,0);\n      ZeroMemory(this.matrices[i].Matrix);\n    }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|   Tensorflows for Vector type of data                            |\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nclass CVectors\n  {\n   public:\n          vector Vector;\n  };\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nclass C2DTensor\n  {\nCVectors             *vectors[];\n\npublic:\n                     C2DTensor(void);\n                    ~C2DTensor(void);\n                     \n                     bool   Init(uint size);\n                     bool Append(vector &v);\n                     \n                     void Print_(void);\n                     CVectors* operator[](const int index) { return GetObj(index); }\n                     CVectors *GetObj(int index);\n                     \n                     void Delete();\n                     uint   Size(); //returns tensor's size\n  };\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nC2DTensor::C2DTensor(void)\n {\n   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nC2DTensor::~C2DTensor(void)\n {\n   for (uint i=0; i<vectors.Size(); i++)\n     if (CheckPointer(vectors[i]) != POINTER_INVALID)\n       delete vectors[i];\n\n   ArrayFree(vectors);\n } \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool C2DTensor::Init(uint size)\n {\n   if (size==0)\n     return false;\n     \n   ArrayResize(this.vectors, size);\n   return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCVectors *C2DTensor::GetObj(int index)\n {\n   if (index<-1 || index > int(vectors.Size()))\n    {\n      printf(\"%s failed, index out of range. Line %d\",__FUNCTION__, __LINE__);\n      return this.vectors[index==-1?vectors.Size()-1: index];\n    }\n   \n   return this.vectors[index==-1?vectors.Size()-1: index]; //if the selected position is -1 we obtain the last matrix in our tensor\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid C2DTensor::Print_(void)\n {\n   for (ulong i=0; i<vectors.Size(); i++)\n     Print(\"TENSOR INDEX [\",i,\"] vector-size =(\",this.vectors[i].Vector.Size(),\")\\n\",this.vectors[i].Vector); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nuint C2DTensor::Size(void)\n {\n   return this.vectors.Size();\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid C2DTensor::Delete(void)\n {\n   for (ulong i=0; i<vectors.Size(); i++)\n    {\n      this.vectors[i].Vector.Resize(0,0);\n      ZeroMemory(this.vectors[i].Vector);\n    }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool C2DTensor::Append(vector &v)\n {\n   if (ArrayResize(this.vectors, vectors.Size()+1)<0)\n    return false;\n    \n   uint SIZE = vectors.Size();\n   vectors[SIZE-1] = new CVectors();\n   vectors[SIZE-1].Vector = v; //Add the new matrix to the newly created tensor index\n   \n   return true;   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+"
  },
  {
    "path": "Utils.mqh",
    "content": "//+------------------------------------------------------------------+\n//|                                                 matrix_utils.mqh |\n//|                                  Copyright 2022, Omega Joctan  . |\n//|                        https://www.mql5.com/en/users/omegajoctan |\n//+------------------------------------------------------------------+\n#property copyright \"Copyright 2022, Omega Joctan\"\n#property link      \"https://www.mql5.com/en/users/omegajoctan\"\n\n//+------------------------------------------------------------------+\n//|   A class containing additional matrix manipulation functions    |\n//+------------------------------------------------------------------+\n\nclass CUtils\n  {\n  \nprotected:\n\n   template<typename T>\n   static T          MathRandom(T mini, T maxi);\n   static string     CalcTimeElapsed(double seconds);\n   static void       Swap(double &var1, double &var2);\n   static string     ConvertTime(double seconds);\n   \n   template<typename T>\n   static void       GetCol(const T &Matrix[], T &Col[], int column, int cols);\n   \n   static bool       IsNumber(string text);\n\npublic:\n                     CUtils(void);\n                    ~CUtils(void);\n   \n   template<typename T>\n   static int Sign(T var)\n    {\n      if (var<0)\n        return -1;\n      else if (var==0)\n        return 0;\n      else\n        return 1;\n    }\n    \n//--- File Functions\n\n   template <typename T>\n   static bool       WriteCsv(string csv_name, matrix<T> &matrix_, string &header[] ,bool common=false, int digits=5);\n   template <typename T>\n   static bool       WriteCsv(string csv_name, matrix<T> &matrix_, string header_string=\"\",bool common=false, int digits=5);\n   static matrix     ReadCsv(string file_name, string &headers, string delimiter=\",\",bool common=false, bool auto_encode=false);\n   static matrix     DBtoMatrix(int db_handle, string table_name,string &column_names[],int total=WHOLE_ARRAY);\n   static bool       write_bin(vector &v, string file);\n   \n//--- Manipulations\n   \n   template<typename T>\n   static bool       RemoveCol(matrix<T> &mat, ulong col);\n   static void       RemoveMultCols(matrix &mat, int &cols[]);\n   static void       RemoveMultCols(matrix &mat, int from, int total=WHOLE_ARRAY);\n   static void       RemoveRow(matrix &mat,ulong row);\n   static void       VectorRemoveIndex(vector &v, ulong index);  \n   \n//--- Machine Learning \n\n   template<typename T>\n   static bool       XandYSplitMatrices(const matrix<T> &matrix_, matrix<T> &xmatrix, vector<T> &y_vector,int y_column=-1);\n   template <typename T>\n   static void       TrainTestSplitMatrices(const matrix<T> &X, const vector<T> &y, matrix<T> &x_train, vector<T> &y_train, matrix<T> &x_test, vector<T> &y_test, double train_size=0.7,int random_state=-1, bool shuffle=true);\n   static matrix     DesignMatrix(const matrix &x);              \n   static matrix     OneHotEncoding(const vector &v);    //ONe hot encoding \n   static matrix     Sign(matrix &x);\n   static vector     Sign(vector &x);\n   \n//--- Detection\n\n   static void       Unique(const string &Array[], string &classes_arr[]);\n   static vector     Unique(const vector &v);           //Identifies classes available in a vector\n   static vector     Unique_count(vector &v);\n   \n   template<typename T> \n   static vector     Random(T min, T max, int size,int random_state=-1);          //Generates a random vector of type T sized = size\n   static matrix     Random(double min, double max, ulong rows, ulong cols, int random_state=-1); \n   \n   template<typename T>\n   static vector     Search(const vector<T> &v, T value);\n   \n//--- Transformations\n\n   static matrix     VectorToMatrix(const vector &v, ulong cols=1);\n   template<typename T>\n   static vector     MatrixToVector(const matrix<T> &mat);\n   \n   template<typename T>\n   static vector     ArrayToVector(const T &Arr[]);     \n   template<typename T>\n   static bool       VectorToArray(const vector<T> &v,T &arr[]);\n   \n//--- Manipulations\n   \n   template<typename T>\n   static vector     concatenate(const vector<T> &v1, const vector<T> &v2);              //Appends v2 to vector 1\n   static vector     concatenate(const vector &v, const double value);              //Appends double value to a vector \n   static matrix     concatenate(const matrix &mat1, const  matrix &mat2, int axis = 0);\n   template<typename T>\n   static matrix<T>  concatenate(const matrix<T> &mat, const vector<T> &v, int axis=1);\n   \n   template<typename T>\n   static bool       Copy(const vector<T> &src, vector<T> &dst, ulong src_start,ulong total=WHOLE_ARRAY);\n   \n   \n   template<typename T>\n   static void       Reverse(vector<T> &v);\n   template<typename T>\n   static void       Reverse(matrix<T> &mat);\n   \n   static matrix     HadamardProduct(matrix &a, matrix &b);\n   \n   template<typename T>\n   static void       Randomize(vector<T> &v, int random_state=-1, bool replace=false);\n   template<typename T>\n   static void       Randomize(matrix<T> &matrix_,int random_state=-1, bool replace=false);\n   \n   template<typename T>\n   static void       NormalizeDouble_(vector<T> &v, int digits=3);\n   template<typename T>\n   static void       NormalizeDouble_(matrix<T> &mat, int digits=3);\n   \n   static int        CopyBufferVector(int handle, int buff_num, int start_pos,int count, vector &v);\n   static string     Stringfy(vector &v, int digits = 2);\n   static matrix     Zeros(ulong rows, ulong cols) { matrix ret_mat(rows, cols); return(ret_mat.Fill(0.0)); }\n   static vector     Zeros(ulong size) { vector ret_v(size); return( ret_v.Fill(0.0)); }\n   static matrix     Get(const matrix &mat, ulong start_index, ulong end_index);\n   static vector     Get(const vector &v, ulong start_index, ulong end_index);\n   template<typename T>\n   static vector     Sort(vector<T> &v,ENUM_SORT_MODE sort_mode=SORT_ASCENDING);\n   template<typename T>\n   static vector     ArgSort(vector<T> &v);\n   static matrix     Slice(const matrix &mat, ulong start_index, int end_index, uint axis=0);\n   static vector     Slice(const vector &vec, ulong start_index, int end_index);\n\n//--- Others\n   \n   static void       PrintShort(matrix &matrix_,ulong rows=5, int digits=5);\n\n  }; \n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCUtils::CUtils(void)\n  {\n    \n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nCUtils::~CUtils(void)\n  {\n  \n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::VectorToMatrix(const vector &v, ulong cols=1)\n  {      \n   ulong rows = 0;\n   matrix mat = {};\n   \n   \n    if ( v.Size() % cols > 0) //If there is a reminder\n      {\n        printf(\"Invalid rows %d and cols %d for this vector size \",rows,v.Size()/cols);\n        return mat;\n      }\n    else\n       rows = v.Size()/cols;\n\n//---\n\n   mat.Resize(rows, cols); \n\n   for(ulong i=0, index =0; i<rows; i++)\n      for(ulong j=0; j<cols; j++, index++)\n        {\n         mat[i][j] = v[index];\n        }\n   return(mat);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::MatrixToVector(const matrix<T> &mat)\n  {\n    vector<T> v = {};\n    matrix<T> temp_mat = mat;\n    \n    if (!temp_mat.Swap(v))\n      Print(__FUNCTION__,\" Failed to turn the matrix[\",mat.Rows(),\"x\",mat.Cols(),\"] into a vector\");\n    \n    return(v);\n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nbool CUtils::RemoveCol(matrix<T> &mat, ulong col)\n  {\n   matrix<T> new_matrix(mat.Rows(),mat.Cols()-1); //Remove the one Column\n   if (col > mat.Cols())\n     {\n       Print(__FUNCTION__,\" column out of range\");\n       return false;\n     }\n\n   for (ulong i=0, new_col=0; i<mat.Cols(); i++) \n     {\n        if (i == col)\n          continue;\n        else\n          {\n           new_matrix.Col(mat.Col(i),new_col);\n           new_col++;\n          }    \n     }\n\n   mat.Copy(new_matrix);\n   \n   return true;\n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::RemoveMultCols(matrix &mat, int &cols[])\n  {\n   ulong size = (int)ArraySize(cols);\n\n   if(size > mat.Cols())\n     {\n      Print(__FUNCTION__,\" Columns to remove can't be more than the available columns\");\n      return;\n     }\n\n\n   vector Zeros(mat.Rows());\n   Zeros.Fill(0);\n\n   for(ulong i=0; i<size; i++)\n      for(ulong j=0; j<mat.Cols(); j++)\n        {\n         if(cols[i] == j)\n            mat.Col(Zeros,j);\n        }\n\n//---\n\n   vector column_vector;\n   \n   while (mat.Cols()-size >= size)\n      for(ulong i=0; i<mat.Cols(); i++)\n        {\n         column_vector = mat.Col(i);\n         if(column_vector.Sum()==0)\n            if (!RemoveCol(mat,i))\n              {\n                printf(\"%s Line %d Failed to remove a column %d from a matrix\",__FUNCTION__,__LINE__,i);\n                break;\n              }\n        }\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\nvoid CUtils::RemoveMultCols(matrix &mat, int from, int total=WHOLE_ARRAY)\n {\n   \n   total = total==WHOLE_ARRAY ? (int)mat.Cols()-from : total;\n   \n   if(total > (int)mat.Cols())\n     {\n      Print(__FUNCTION__,\" Columns to remove can't be more than the available columns\");\n      return;\n     }\n\n   vector Zeros(mat.Rows());\n   Zeros.Fill(0);\n\n   for (int i=from; i<total+from; i++)\n      mat.Col(Zeros, i);\n   \n//---      \n   \n   ulong remain_size = mat.Cols()-total;\n   \n   \n   while (mat.Cols() >= remain_size && !IsStopped())\n    {\n      //printf(\"cols %d total %d\",cols,total);\n      \n      for(ulong i=0; i<mat.Cols(); i++) //loop the entire matrix searching for columns to remove\n         if(mat.Col(i).Sum()==0)\n            if (!RemoveCol(mat,i))\n              {\n                printf(\"%s Line %s Failed to remove a column %d from a matrix\",__FUNCTION__,__LINE__,i);\n                break;\n              }\n    }\n }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::RemoveRow(matrix &mat,ulong row)\n  {\n   matrix new_matrix(mat.Rows()-1,mat.Cols()); //Remove the one Row\n \n      for(ulong i=0, new_rows=0; i<mat.Rows(); i++)\n        {\n         if(i == row)\n            continue;\n         else\n           {\n            new_matrix.Row(mat.Row(i),new_rows);\n            new_rows++;\n           }\n        }\n\n   mat.Copy(new_matrix);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::VectorRemoveIndex(vector &v, ulong index)\n  {\n   vector new_v(v.Size()-1);\n\n   for(ulong i=0, count = 0; i<v.Size(); i++)\n      if(i != index)\n        {\n         new_v[count] = v[i];\n         count++;\n        }\n    v.Copy(new_v);\n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate <typename T>\nbool CUtils::WriteCsv(string csv_name, matrix<T> &matrix_, string &header[], bool common=false, int digits=5)\n  {\n   string header_str = \"\";\n   for (int i=0; i<ArraySize(header); i++)\n      header_str += header[i] + ((i+1 == ArraySize(header)) ? \"\" : \",\");\n      \n   return WriteCsv(csv_name, matrix_, header_str, common, digits);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate <typename T>\nbool CUtils::WriteCsv(string csv_name, matrix<T> &matrix_, string header_string=\"\", bool common=false, int digits=5)\n  {\n   FileDelete(csv_name);\n   int handle = FileOpen(csv_name,FILE_WRITE|FILE_CSV|FILE_ANSI|(common?FILE_COMMON:FILE_IS_WRITABLE),\",\",CP_UTF8);\n   \n   if (header_string == \"\" || header_string == NULL)\n     for (ulong i=0; i<matrix_.Cols(); i++)\n       header_string += \"None\"+ (i==matrix_.Cols()-1?\"\":\",\"); \n\n   if(handle == INVALID_HANDLE)\n     {\n       printf(\"Invalid %s handle Error %d \",csv_name,GetLastError());\n       return (false);\n     }\n            \n   string concstring;\n   vector<T> row = {};\n   \n   datetime time_start = GetTickCount(), current_time;\n   \n   string header[];\n   \n   ushort u_sep;\n   u_sep = StringGetCharacter(\",\",0);\n   StringSplit(header_string,u_sep, header);\n   \n   vector<T> colsinrows = matrix_.Row(0);\n   \n   if (ArraySize(header) != (int)colsinrows.Size())\n      {\n         printf(\"headers=%d and columns=%d from the matrix vary is size \",ArraySize(header),colsinrows.Size());\n         return false;\n      }\n\n//---\n\n   string header_str = \"\";\n   for (int i=0; i<ArraySize(header); i++)\n      header_str += header[i] + (i+1 == colsinrows.Size() ? \"\" : \",\");\n   \n   FileWrite(handle,header_str);\n   \n   FileSeek(handle,0,SEEK_SET);\n   \n   for(ulong i=0; i<matrix_.Rows() && !IsStopped(); i++)\n     {\n      ZeroMemory(concstring);\n\n      row = matrix_.Row(i);\n      for(ulong j=0, cols =1; j<row.Size() && !IsStopped(); j++, cols++)\n        {\n         current_time = GetTickCount();\n         \n         Comment(\"Writting \",csv_name,\" record [\",i+1,\"/\",matrix_.Rows(),\"] Time taken | \",ConvertTime((current_time - time_start) / 1000.0));\n         \n         concstring += (string)NormalizeDouble(row[j],digits) + (cols == matrix_.Cols() ? \"\" : \",\");\n        }\n\n      FileSeek(handle,0,SEEK_END);\n      FileWrite(handle,concstring);\n     }\n        \n   FileClose(handle);\n   \n   return (true);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CUtils::IsNumber(string text)\n{\n    int length = StringLen(text);   // Get the length of the string.\n    int pointcount = 0;             // Initialize a counter for the number of decimal points.\n\n    // Iterate through each character in the text.\n    for (int i = 0; i < length; i++)\n    {\n        int char1 = StringGetCharacter(text, i);  // Get the ASCII code of the current character.\n\n        // If the character is a decimal point, increment the decimal point counter.\n        if (char1 == 46)\n            pointcount += 1;\n\n        // If the character is a digit or a decimal point and the number of decimal points is less than 2,\n        // continue to the next character; otherwise, return false.\n        if (((char1 >= 48 && char1 <= 57) || char1 == 46) && pointcount < 2)\n            continue;\n        else\n            return false;\n    }\n\n    // If all characters in the text have been checked without returning false, return true.\n    return true;\n}\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::GetCol(const T &Matrix[], T &Col[], int column, int cols)\n {\n   int rows = ArraySize(Matrix)/cols;\n   ArrayResize(Col,rows);\n   \n   int start = 0;\n    for (int i=0; i<cols; i++)\n     {\n      start = i;\n      \n      if (i != column-1)  continue;\n      else\n        for (int j=0; j<rows; j++)\n          {\n            //printf(\"ColMatrix[%d} Matrix{%d]\",j,start);\n            Col[j] = Matrix[start];\n            \n            start += cols;\n          }\n     }  \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::ArrayToVector(const T &Arr[])\n  {\n   vector v(ArraySize(Arr));\n   \n   for (int i=0; i<ArraySize(Arr); i++)\n     v[i] = double(Arr[i]);\n     \n   return (v);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nbool CUtils::VectorToArray(const vector<T> &v, T &arr[])\n  {\n   vector temp = v;\n   if (!temp.Swap(arr))\n    {\n      Print(\"Failed to Convert vector to Array Err=\",GetLastError());\n      return false;\n    }\n   return(true);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nbool CUtils::XandYSplitMatrices(const matrix<T> &matrix_, matrix<T> &xmatrix, vector<T> &y_vector,int y_column=-1)\n  {\n   y_column = int( y_column==-1 ? matrix_.Cols()-1 : y_column);\n   \n   if (matrix_.Rows() == 0 || matrix_.Cols()==0)\n     {\n       #ifdef DEBUG_MODE\n         printf(\"%s Line %d Cannot split the matrix of size[%dx%d]\",__FUNCTION__,__LINE__,matrix_.Rows(),matrix_.Cols());\n       #endif \n       \n       return false;\n     }\n   \n   y_vector = matrix_.Col(y_column);\n   xmatrix.Copy(matrix_);\n   \n   return RemoveCol(xmatrix, y_column); //Remove the y column\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::Randomize(vector<T> &v, int random_state=-1, bool replace=false)\n {\n   MathSrand(random_state!=-1?random_state:GetTickCount());\n     \n   int swap_index;\n   double temp;\n   \n   int SIZE = (int)v.Size();\n   vector<T> temp_v = v;\n   \n   for (int i=0; i<SIZE; i++) //Fisher yates algorithm\n      {\n        if (!replace)\n          {\n            swap_index = rand() % SIZE;\n            \n            temp = v[i];\n            \n            v[i] = v[swap_index];\n            v[swap_index] = temp;\n          }\n        else\n          {\n            v[i] = temp_v[MathRandom(0, SIZE)];\n          }\n      }   \n }\n//+------------------------------------------------------------------+\n//| replace =true parameter allows the same index to be chosen more  |\n//| than once, simulating the bootstrapping process.                 |    \n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::Randomize(matrix<T> &matrix_,int random_state=-1, bool replace=false)\n {\n   MathSrand(random_state!=-1?random_state:GetTickCount());\n  \n   int ROWS=(int)matrix_.Rows(), COL=(int)matrix_.Cols();   \n   \n   int swap_index;\n   vector<T> temp(COL);\n   matrix<T> temp_m = matrix_;\n   int random = 0;\n   \n   for (int i=0; i<ROWS; i++)\n      {\n        if (!replace)\n          {\n            swap_index = MathRand() % ROWS;\n            \n            temp = matrix_.Row(i);\n                  \n            matrix_.Row(matrix_.Row(swap_index),i);\n            \n            matrix_.Row(temp,swap_index);\n          }\n        else\n          {\n            random = MathRandom(1, ROWS);  \n            \n            temp = temp_m.Row(random-1);                      \n            matrix_.Row(temp, i);\n          }\n      }   \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate <typename T>\nvoid CUtils::TrainTestSplitMatrices(const matrix<T> &X, const vector<T> &y, matrix<T> &x_train, vector<T> &y_train, matrix<T> &x_test, vector<T> &y_test, double train_size=0.7,int random_state=-1, bool shuffle=true)\n  {\n   ulong total = X.Rows(), cols = X.Cols();\n   \n   ulong last_col = cols-1;\n   \n//--- Random pseudo matrix\n   \n   matrix temp_x = X; vector temp_y = y;\n   \n   if (shuffle)\n    {\n      matrix temp_matrix = concatenate(X, y);\n      Randomize(temp_matrix, random_state);\n      XandYSplitMatrices(temp_matrix, temp_x, temp_y);\n    }\n    \n//--- Resizing the new metrices\n\n   int train = (int)MathFloor(total*train_size);\n   int test = (int)total-train;\n   \n   \n//---\n   \n   x_train = Slice(temp_x, 0, train);\n   x_test = Slice(temp_x, train, -1);\n   \n//---\n   \n   y_train = Slice(temp_y, 0, train);\n   y_test = Slice(temp_y, train, -1);\n   \n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::DesignMatrix(const matrix &x)\n  {\n   matrix out_matrix(x.Rows(),x.Cols()+1);\n\n   vector ones(x.Rows());\n   ones.Fill(1);\n\n   out_matrix.Col(ones,0);\n   vector new_vector;\n\n   for(ulong i=1; i<out_matrix.Cols(); i++)\n     {\n      new_vector = x.Col(i-1);\n      out_matrix.Col(new_vector,i);\n     }\n\n   return (out_matrix);\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::OneHotEncoding(const vector &v)\n {\n   matrix mat = {}; \n   \n//---\n\n   vector v_classes = Unique(v);\n     \n//---\n\n     mat.Resize(v.Size(),v_classes.Size());\n     mat.Fill(-100);\n     \n     for (ulong i=0; i<mat.Rows(); i++)\n        for (ulong j=0; j<mat.Cols(); j++)\n           {\n               if (v[i] == v_classes[j])\n                  mat[i][j] = 1;\n               else \n                  mat[i][j] = 0;     \n           }\n   \n   return(mat);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::Unique(const string &Array[], string &classes_arr[])\n {\n   string temp_arr[];\n\n   ArrayResize(classes_arr,1);\n   ArrayCopy(temp_arr,Array);\n   \n   classes_arr[0] = Array[0];\n   \n   for(int i=0, count =1; i<ArraySize(Array); i++)  //counting the different neighbors\n     {\n      for(int j=0; j<ArraySize(Array); j++)\n        {\n         if(Array[i] == temp_arr[j] && temp_arr[j] != \"-nan\")\n           {\n            bool count_ready = false;\n\n            for(int n=0; n<ArraySize(classes_arr); n++)\n               if(Array[i] == classes_arr[n])\n                    count_ready = true;\n\n            if(!count_ready)\n              {\n               count++;\n               ArrayResize(classes_arr,count);\n\n               classes_arr[count-1] = Array[i]; \n\n               temp_arr[j] = \"-nan\"; //modify so that it can no more be counted\n              }\n            else\n               break;\n           }\n         else\n            continue;\n        }\n     }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CUtils::Unique(const vector &v)\n {\n   vector temp_v = v; \n   vector v_classes={v[0]};\n\n   for (ulong i = 0, count=0; i < v.Size(); i++) \n    {\n      bool alreadyCounted = false;\n\n      for (ulong j = 0; j < v_classes.Size(); j++) \n       {\n         if (temp_v[i] == v_classes[j] && temp_v[i] != -DBL_MAX && i!=0) \n           {\n             alreadyCounted = true;\n             temp_v[i] = -DBL_MAX;\n           }\n      }\n\n     if (!alreadyCounted) \n       {\n         count++;\n         v_classes.Resize(count);\n         \n         v_classes[count-1] = temp_v[i];\n       }\n    }\n \n   return CUtils::Sort(v_classes); //Sort the unique values in ascending order\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nT CUtils:: MathRandom(T mini, T maxi)\n  {\n     double  f  = (MathRand() / 32767.0);\n     return (mini + (T)(f * (maxi - mini)));\n  }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T> \nvector CUtils::Random(T min, T max,int size,int random_state=-1)\n {\n   MathSrand(random_state!=-1?random_state:GetTickCount());\n    \n   vector v(size);\n   \n   for (ulong i=0; i<v.Size(); i++)\n      v[i] = MathRandom<T>(min,max);\n      \n   return (v);    \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::Random(double min,double max,ulong rows,ulong cols,int random_state=-1)\n {\n   MathSrand(random_state!=-1?random_state:GetTickCount());\n     \n     matrix mat(rows,cols);\n     \n     for (ulong r=0; r<rows; r++)\n       for (ulong c=0; c<cols; c++)\n            mat[r][c] = MathRandom<double>(min,max);\n     \n     return (mat);\n }\n//+------------------------------------------------------------------+\n//|   Appends vector v1 to the end of vector v2                      |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::concatenate(const vector<T> &v1, const vector<T> &v2)\n {\n   vector v_out = v1; \n   \n   v_out.Resize(v1.Size()+v2.Size());\n   \n   for (ulong i=0; i<v1.Size(); i++)\n      v_out[i] = v1[i]; \n   \n   for (ulong i=v1.Size(),index =0; i<v_out.Size(); i++, index++)\n       v_out[i] = v2[index]; \n   \n   return (v_out); \n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CUtils::concatenate(const vector &v, const double value) //Appends double value to a vector \n {\n   vector v2 = {value};\n   return concatenate(v, v2);\n }\n//+------------------------------------------------------------------+\n//|   Appends matrix mat1 to the end of mat2                         |\n//+------------------------------------------------------------------+\nmatrix CUtils::concatenate(const matrix &mat1, const matrix &mat2, int axis = 0)\n {\n     matrix m_out = {};\n\n     if ((axis == 0 && mat1.Cols() != mat2.Cols() && mat1.Cols()>0) || (axis == 1 && mat1.Rows() != mat2.Rows() && mat1.Rows()>0)) \n       {\n         Print(__FUNCTION__, \"Err | Dimensions mismatch for concatenation\");\n         return m_out;\n       }\n\n     if (axis == 0) {\n         m_out.Resize(mat1.Rows() + mat2.Rows(), MathMax(mat1.Cols(), mat2.Cols()));\n\n         for (ulong row = 0; row < mat1.Rows(); row++) {\n             for (ulong col = 0; col < m_out.Cols(); col++) {\n                 m_out[row][col] = mat1[row][col];\n             }\n         }\n\n         for (ulong row = 0; row < mat2.Rows(); row++) {\n             for (ulong col = 0; col < m_out.Cols(); col++) {\n                 m_out[row + mat1.Rows()][col] = mat2[row][col];\n             }\n         }\n     } else if (axis == 1) {\n         m_out.Resize(MathMax(mat1.Rows(), mat2.Rows()), mat1.Cols() + mat2.Cols());\n\n         for (ulong row = 0; row < m_out.Rows(); row++) {\n             for (ulong col = 0; col < mat1.Cols(); col++) {\n                 m_out[row][col] = mat1[row][col];\n             }\n\n             for (ulong col = 0; col < mat2.Cols(); col++) {\n                 m_out[row][col + mat1.Cols()] = mat2[row][col];\n             }\n         }\n     }\n   return m_out;\n }\n//+------------------------------------------------------------------+\n//|   Concatenates the vector to a matrix, axis =0 along the rows    |\n//|   while axis =1 along the colums concatenation\n//+------------------------------------------------------------------+\ntemplate<typename T>\nmatrix<T> CUtils::concatenate(const matrix<T> &mat, const vector<T> &v, int axis=1)\n {\n   matrix<T> ret= mat;\n     \n   ulong new_rows, new_cols;\n   \n   if (axis == 0) //place it along the rows\n    {\n      if (mat.Cols() == 0)\n        ret.Resize(mat.Rows(), v.Size());\n        \n      new_rows = ret.Rows()+1; new_cols = ret.Cols();\n                 \n      if (v.Size() != new_cols)\n        {\n          Print(__FUNCTION__,\" Dimensions don't match the vector v needs to have the same size as the number of columns in the original matrix\");\n          return ret;\n        }\n      \n      ret.Resize(new_rows, new_cols);\n      ret.Row(v, new_rows-1);\n    }\n   else if (axis == 1)\n     {\n         if (mat.Rows() == 0)\n           ret.Resize(v.Size(), ret.Cols());\n           \n        new_rows = ret.Rows(); new_cols = ret.Cols()+1;\n        \n        if (v.Size() != new_rows)\n          {\n            Print(__FUNCTION__,\" Dimensions don't match the vector v needs to have the same size as the number of rows in the original matrix\");\n            return ret;\n          }\n        \n        ret.Resize(new_rows, new_cols);\n        ret.Col(v, new_cols-1);\n     }\n   else \n     {\n       Print(__FUNCTION__,\" Axis value Can either be 0 or 1\");\n       return ret;\n     }\n\n//---\n   return ret;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nbool CUtils::Copy(const vector<T> &src, vector<T> &dst,ulong src_start,ulong total=WHOLE_ARRAY)\n {\n   if (total == WHOLE_ARRAY)\n      total = src.Size()-src_start;\n   \n   if ( total <= 0 || src.Size() == 0)\n    {\n       printf(\"%s Can't copy a vector | Size %d total %d src_start %d \",__FUNCTION__,src.Size(),total,src_start);\n       return (false);\n    }\n   \n   dst.Resize(total);\n   dst.Fill(0);\n   \n   for (ulong i=src_start, index =0; i<total+src_start; i++)\n      {   \n          dst[index] = src[i];         \n          index++;\n      }\n   return (true);\n }\n//+------------------------------------------------------------------+\n//| Searches for a value in a vector | Returns all the index where   |\n//| Such values was located                                          |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::Search(const vector<T> &v, T value)\n {\n   vector<T> v_out ={};\n   \n   for (ulong i=0, count =0; i<v.Size(); i++)\n     { \n      if (value == v[i])\n        {\n          count++;\n          \n          v_out.Resize(count);    \n          \n          v_out[count-1] = (T)i;\n        }\n     }   \n    return v_out;\n }\n//+------------------------------------------------------------------+\n//| Finds the unique values in a vector and returns a vector of      |\n//| the number of values found for each unique value                 |\n//+------------------------------------------------------------------+\nvector CUtils::Unique_count(vector &v)\n {\n  vector classes = CUtils::Unique(v);\n  vector keys(classes.Size());\n  \n   for (ulong i=0; i<classes.Size(); i++)\n     keys[i] = (int)Search(v, classes[i]).Size();\n    \n  return keys;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::Sort(vector<T> &v,ENUM_SORT_MODE sort_mode=SORT_ASCENDING)\n {\n   T arr[];\n   vector temp = v;\n   temp.Swap(arr);\n   \n   if (!ArraySort(arr))\n     printf(\"%s Failed to sort this vector Err=%d\",__FUNCTION__,GetLastError());\n   \n   switch(sort_mode)\n     {\n      case  SORT_ASCENDING:\n        temp = CUtils::ArrayToVector(arr);  \n        break;\n      case SORT_DESCENDING:\n        temp = CUtils::ArrayToVector(arr);  \n        CUtils::Reverse(temp);\n        break;\n      default:\n        printf(\"%s Unknown sort mode\");\n        break;\n     }\n   return temp;   \n }\n//+------------------------------------------------------------------+\n//| Returns the Sorted Argsuments in either ascending order or       |\n//|  descending order                                                |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvector CUtils::ArgSort(vector<T> &v)\n {   \n//---\n\n    ulong size = v.Size();\n    vector args(size);\n    \n    // Initialize args array with sequential values\n    for (ulong i = 0; i < size; i++)\n        args[i] = (int)i;\n\n    // Perform selection sort on args based on array values\n    for (ulong i = 0; i < size - 1; i++)\n    {\n        ulong minIndex = i;\n        for (ulong j = i + 1; j < size; j++)\n        {\n            if (v[(int)args[j]] < v[(int)args[minIndex]])\n                minIndex = j;\n        }\n\n        // Swap args\n        int temp = (int)args[i];\n        args[i] = args[minIndex];\n        args[minIndex] = temp;\n    }\n   \n  return args;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::Reverse(vector<T> &v)\n {\n  vector<T> v_temp = v;\n  \n   for (ulong i=0, j=v.Size()-1; i<v.Size(); i++, j--)\n        v[i] = v_temp[j];\n        \n   ZeroMemory(v_temp);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::Reverse(matrix<T> &mat)\n {\n   matrix<T> temp_mat = mat;\n   \n   for (ulong i=0, j=mat.Rows()-1; i<mat.Rows(); i++, j--)\n      mat.Row(mat.Row(j), i); \n }\n//+------------------------------------------------------------------+\n//|   Hadamard product --> is a binary operation that takes two      |\n//|    matrices of the same dimensions and produces another matrix   |\n//|   of the same dimension as the operands. | This operation is     |\n//|  widely known as element wise multiplication                     |\n//+------------------------------------------------------------------+\nmatrix CUtils::HadamardProduct(matrix &a,matrix &b)\n {  \n  matrix c = {};\n  if (a.Rows() != b.Rows() || a.Cols() != b.Cols())\n    {\n      Print(\"Cannot calculate Hadamard product | matrix a and b are not having the same size \");\n      return c;\n    }\n//---\n         \n    return a*b;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n\n\nstring CUtils::CalcTimeElapsed(double seconds)\n  {\n   string time_str = \"\";\n\n   uint minutes=0, hours=0;\n\n   if(seconds >= 60)\n      time_str = StringFormat(\"%d Minutes and %.3f Seconds \",minutes=(int)round(seconds/60.0), ((int)seconds % 60));\n   if(minutes >= 60)\n      time_str = StringFormat(\"%d Hours %d Minutes and %.3f Seconds \",hours=(int)round(minutes/60.0), minutes, ((int)seconds % 60));\n   else\n      time_str = StringFormat(\"%.3f Seconds \",seconds);\n\n   return time_str;\n  }\n\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::DBtoMatrix(int db_handle, string table_name,string &column_names[],int total=WHOLE_ARRAY)\n {\n  matrix matrix_ = {};\n  \n  \n  #ifdef DEBUG_MODE\n     Print(\"---> loading database \");\n  #endif \n  \n  if(db_handle == INVALID_HANDLE)\n     {\n      printf(\"db handle failed, Err = %d\",GetLastError());\n      DatabaseClose(db_handle);\n      return matrix_;\n     }\n\n//---\n\n   string sql =  \"SELECT * FROM \"+table_name;\n   int request = DatabasePrepare(db_handle,sql);     \n   \n   ulong cols = DatabaseColumnsCount(request), rows =0;\n   \n   ArrayResize(column_names,(int)cols);\n\n//---\n\n   matrix_.Resize(cols,0); \n    \n    double time_start = GetMicrosecondCount()/(double)1e6, time_stop=0; //Elapsed time \n    double row_start = 0, row_stop =0;\n     \n    for (int j=0; DatabaseRead(request) && !IsStopped(); j++)\n      {  \n        \n       row_start = GetMicrosecondCount()/(double)1e6; \n           \n        rows = (ulong)j+1;\n        matrix_.Resize(cols,rows);\n         \n         for (ulong k=0; k<cols; k++)\n           {\n             DatabaseColumnDouble(request,(int)k,matrix_[k][j]);\n             \n             if (j==0)  DatabaseColumnName(request,(int)k,column_names[k]);\n           }\n          \n         if (total != WHOLE_ARRAY)\n            if (j >= total)     break;\n            \n         #ifdef  DEBUG_MODE\n            row_stop =GetMicrosecondCount()/(double)1e6;\n            \n            printf(\"Row ----> %d | Elapsed %s\",j,CalcTimeElapsed(row_stop-row_start));\n         #endif \n      }\n\n//---\n   \n   DatabaseFinalize(request);\n   DatabaseClose(db_handle);\n   \n   matrix_ = matrix_.Transpose(); //very crucial step\n   \n   #ifdef DEBUG_MODE\n      time_stop = GetMicrosecondCount()/(double)1e6;\n     \n      printf(\"---> finished reading DB size=(%dx%d) | Time Elapsed %s\",rows,cols,CalcTimeElapsed(time_stop-time_start)); \n     \n      ArrayPrint(column_names);\n      for (ulong i=0; i<5; i++)     Print(matrix_.Row(i));\n   #endif \n   \n   \n  return matrix_;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::NormalizeDouble_(vector<T> &v,int digits=3)\n {\n   for (ulong i=0; i<v.Size(); i++)\n      v[i] = NormalizeDouble(v[i], digits);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\ntemplate<typename T>\nvoid CUtils::NormalizeDouble_(matrix<T> &mat,int digits=3)\n {\n   for (ulong i=0; i<mat.Rows(); i++)\n      for (ulong j=0; j<mat.Cols(); j++)\n         mat[i][j] = NormalizeDouble(mat[i][j], digits);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::PrintShort(matrix &matrix_, ulong rows=5,int digits=5)\n {\n   vector v = {};\n    for (ulong i=0; i<rows; i++)\n     {\n        v = matrix_.Row(i);\n        NormalizeDouble_(v, digits);\n        \n        Print(v); \n     }\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvoid CUtils::Swap(double &var1,double &var2)\n {\n   double temp_1 = var1, temp2=var2;\n   \n   var1 = temp2;\n   var2 = temp_1;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nint CUtils::CopyBufferVector(int handle,int buff_num,int start_pos,int count,vector &v)\n {\n   double buff_arr[];\n   \n   int ret = CopyBuffer(handle, buff_num, start_pos, count, buff_arr);\n   v = ArrayToVector(buff_arr);\n   \n   return (ret);\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nstring CUtils::Stringfy(vector &v, int digits = 2)\n {\n   string str = \"\";\n   for (ulong i=0; i<v.Size(); i++)\n       str += \" \"+DoubleToString(v[i], digits) + \" \";\n   \n   return (str);\n }\n//+------------------------------------------------------------------+\n//| a function to convert the seocnds to Hours and minutes, Useful   |\n//| in measuring the time taken for operations that takes a long     |\n//| time to complete, Such as reading and writing to a large csv file|\n//+------------------------------------------------------------------+\nstring CUtils::ConvertTime(double seconds)\n{\n    string time_str = \"\";\n    uint minutes = 0, hours = 0;\n\n    if (seconds >= 60)\n    {\n        minutes = (uint)(seconds / 60.0) ;\n        seconds = fmod(seconds, 1.0) * 60;\n        time_str = StringFormat(\"%d Minutes and %.3f Seconds\", minutes, seconds);\n    }\n    \n    if (minutes >= 60)\n    {\n        hours = (uint)(minutes / 60.0);\n        minutes = minutes % 60;\n        time_str = StringFormat(\"%d Hours and %d Minutes\", hours, minutes);\n    }\n\n    if (time_str == \"\")\n    {\n        time_str = StringFormat(\"%.3f Seconds\", seconds);\n    }\n\n    return time_str;\n}\n//+------------------------------------------------------------------+\n//|  Obtains a part of the matrix starting from a start_index row to |\n//|   end_index row Inclusive                                        |\n//+------------------------------------------------------------------+\nmatrix CUtils::Get(const matrix &mat, ulong start_index, ulong end_index)\n {\n  matrix ret_mat(MathAbs(end_index-start_index+1), mat.Cols());\n  \n  if (start_index >= mat.Rows())\n    {\n       Print(__FUNCTION__,\" Error | start_index (\",start_index,\") is greater than or Equal to matrix Rows (\",mat.Rows(),\")\");\n       return ret_mat;\n    }\n    \n  if (end_index > mat.Rows())\n   {\n       Print(__FUNCTION__,\" Error | end_index (\",end_index,\") is greater than (\",mat.Rows(),\")\");\n       return ret_mat;\n   }\n  \n  if (start_index > end_index)\n    {\n      Print(__FUNCTION__,\" Error | start_index shouldn't be greater than end_index ???\");\n      return ret_mat;\n    }\n  \n   for (ulong i=start_index, count =0; i<=end_index; i++, count++)\n     for (ulong col=0; col<mat.Cols(); col++)\n         ret_mat[count][col] = mat[i][col];\n       \n   return ret_mat;\n }\n\n//+------------------------------------------------------------------+\n//|  Obtains a part of the vector starting from a start_index row to |\n//|   end_index row Inclusive                                        |\n//+------------------------------------------------------------------+\n\nvector CUtils::Get(const vector &v, ulong start_index, ulong end_index)\n {\n  vector ret_vec(MathAbs(end_index-start_index+1));\n  \n  if (start_index >= v.Size())\n    {\n       Print(__FUNCTION__,\"Error | start_index (\",start_index,\") is greater than or Equal to matrix Rows (\",v.Size(),\")\");\n       return ret_vec;\n    }\n    \n  if (end_index > v.Size())\n   {\n       Print(__FUNCTION__,\"Error | end_index (\",start_index,\") is greater than (\",v.Size(),\")\");\n       return ret_vec;\n   }\n  \n  if (start_index > end_index)\n    {\n      Print(__FUNCTION__,\"Error | start_index shouldn't be greater than end_index ???\");\n      return ret_vec;\n    }\n  \n  for (ulong i=start_index, count=0; i<=end_index; i++, count++)\n     ret_vec[count] = v[i];\n       \n   return ret_vec;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::Sign(matrix &x)\n {\n   matrix ret_matrix = x;\n   \n    for (ulong i=0; i<x.Cols(); i++)\n     ret_matrix.Col(Sign(x.Col(i)) ,i);\n   \n   return ret_matrix;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CUtils::Sign(vector &x)\n {\n   vector v(x.Size());\n   for (ulong i=0; i<x.Size(); i++)\n     v[i] = Sign(x[i]);\n     \n  return v;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nbool CUtils::write_bin(vector &v,string file)\n {\n   FileDelete(file);\n   int handle = FileOpen(file,FILE_READ|FILE_WRITE|FILE_BIN,\",\");\n   if (handle == INVALID_HANDLE)\n    {\n      printf(\"Invalid handle Err=%d\",GetLastError());\n      DebugBreak();\n      return false;\n    }\n   \n   double arr[];\n   ArrayResize(arr, (int)v.Size());\n   \n   for (uint i=0; i<arr.Size(); i++)\n    arr[i] = v[i];\n   \n   FileWriteArray(handle, arr);\n   FileClose(handle);\n  \n  return true;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nmatrix CUtils::Slice(const matrix &mat, ulong start_index, int end_index, uint axis=0)\n {\n   matrix sliced = {};\n   \n   if (end_index<-1)\n     {\n       printf(\"%s line=%d, INvalid end index=%d\",__FUNCTION__,__LINE__, end_index);\n       return sliced;\n     }\n   \n   if (end_index==-1)\n     end_index = (int)mat.Rows();\n\n//---\n\n   if ((int)start_index>=end_index)\n    {\n      printf(\"%s failed, end must be >= start\",__FUNCTION__);\n      return sliced;\n    }    \n   \n   for (ulong i=start_index, count=0; i<(ulong)end_index; i++, count++)\n     {\n         switch(axis)\n           {\n            case  0: //slice along rows\n              \n               if (end_index-start_index>mat.Rows())\n                 {\n                   printf(\"%s failed, Index out of range line %d\",__FUNCTION__,__LINE__);\n                   return sliced;\n                 }\n                 \n                 sliced.Resize(end_index-start_index, mat.Cols());\n                 sliced.Row(mat.Row(i) , count);\n              \n              break;\n            case 1: //slice along columns\n             \n               if (end_index-start_index>mat.Cols())\n                 {\n                   printf(\"%s failed, Index out of range line %d\",__FUNCTION__,__LINE__);\n                   return sliced;\n                 }\n                 \n                 sliced.Resize(mat.Rows(), end_index-start_index);\n                 sliced.Col(mat.Col(i) , count);\n              \n              break;\n            default:\n              printf(\"%s failed, Unknown axis value %d\",__FUNCTION__, axis);\n              return sliced;\n              break;\n           } \n     } \n     \n   return sliced;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\nvector CUtils::Slice(const vector &vec, ulong start_index, int end_index)\n {\n   vector sliced = {};\n   \n   if (end_index<-1)\n     {\n       printf(\"%s line=%d, Invalid end index=%d\",__FUNCTION__,__LINE__, end_index);\n       return sliced;\n     }\n   \n   if (end_index==-1)\n     end_index = (int)vec.Size();\n     \n//---\n   \n   if ((int)start_index>=end_index)\n    {\n      printf(\"%s failed, end must be >= start\",__FUNCTION__);\n      return sliced;\n    }\n    \n\n   sliced.Resize(end_index-start_index);\n   for (ulong i=start_index, count=0; i<(ulong)end_index; i++, count++)\n     {\n         if (end_index-start_index>vec.Size())\n           {\n             printf(\"%s failed, Index out of range line %d\",__FUNCTION__,__LINE__);\n             return sliced;\n           }\n           \n           sliced[count] = vec[i];\n     } \n     \n   return sliced;\n }\n//+------------------------------------------------------------------+\n//|                                                                  |\n//+------------------------------------------------------------------+\n"
  },
  {
    "path": "requirements.txt",
    "content": "contourpy==1.0.7\ncycler==0.11.0\nfonttools==4.38.0\nkiwisolver==1.4.4\nmatplotlib==3.7.0\nnumpy==1.24.2\npackaging==23.0\npandas==1.5.3\nPillow==9.4.0\npyparsing==3.0.9\npython-dateutil==2.8.2\npytz==2022.7.1\nseaborn==0.12.2\nsix==1.16.0\n"
  }
]