Showing preview only (492K chars total). Download the full file or copy to clipboard to get everything.
Repository: MegaJoctan/MALE5
Branch: MQL5-ML
Commit: d6541568aaba
Files: 47
Total size: 473.0 KB
Directory structure:
gitextract_ly_vy9wi/
├── .github/
│ └── FUNDING.yml
├── .gitignore
├── Examples/
│ ├── Classifier Model Example.mq5
│ └── Regressor Model Example.mq5
├── LICENSE
├── MqPlotLib/
│ └── plots.mqh
├── Neural Networks/
│ ├── Pattern Nets.mqh
│ ├── README.md
│ ├── Regressor Nets.mqh
│ ├── initializers.mqh
│ ├── kohonen maps.mqh
│ └── optimizers.mqh
├── Numpy/
│ └── Numpy.mqh
├── Pandas/
│ ├── Incremental LE.mqh
│ └── pandas.mqh
├── README.md
├── Sklearn/
│ ├── Cluster/
│ │ ├── DBSCAN.mqh
│ │ ├── Hierachical Clustering.mqh
│ │ ├── KMeans.mqh
│ │ └── base.mqh
│ ├── Decomposition/
│ │ ├── LDA.mqh
│ │ ├── NMF.mqh
│ │ ├── PCA.mqh
│ │ ├── README.md
│ │ ├── TruncatedSVD.mqh
│ │ └── base.mqh
│ ├── Ensemble/
│ │ ├── AdaBoost.mqh
│ │ ├── README.md
│ │ └── Random Forest.mqh
│ ├── Linear Models/
│ │ ├── Linear Regression.mqh
│ │ ├── Logistic Regression.mqh
│ │ ├── README.md
│ │ └── Ridge.mqh
│ ├── Naive Bayes/
│ │ ├── Naive Bayes.mqh
│ │ ├── README.md
│ │ └── naive bayes visuals.py
│ ├── Neighbors/
│ │ └── KNN_nearest_neighbors.mqh
│ ├── Tree/
│ │ ├── README.md
│ │ └── tree.mqh
│ ├── metrics.mqh
│ └── preprocessing.mqh
├── Stats Models/
│ ├── ADF.mqh
│ ├── ARIMA.mqh
│ └── OLS.mqh
├── Tensors.mqh
├── Utils.mqh
└── requirements.txt
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/FUNDING.yml
================================================
# These are supported funding model platforms
ko_fi: omegajoctan
custom: ['https://www.mql5.com/en/users/omegajoctan/seller']
custom: ['https://www.buymeacoffee.com/omegajoctan']
================================================
FILE: .gitignore
================================================
*.ex5
*.psd
*.zip
*.rar
*.xlsx
Todo's.txt
logisticwiki.txt
/venv
/Neural Nets Pro
================================================
FILE: Examples/Classifier Model Example.mq5
================================================
//+------------------------------------------------------------------+
//| Classifier Model Sample.mq5 |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#property version "1.00"
#include <MALE5\Decision Tree\tree.mqh>
#include <MALE5\preprocessing.mqh>
#include <MALE5\MatrixExtend.mqh> //helper functions for for data manipulations
#include <MALE5\metrics.mqh> //fo measuring the performance
StandardizationScaler scaler; //standardization scaler from preprocessing.mqh
CDecisionTreeClassifier *decision_tree; //a decision tree classifier model
MqlRates rates[];
//+------------------------------------------------------------------+
//| Expert initialization function |
//+------------------------------------------------------------------+
int OnInit()
{
//--- Model selection
decision_tree = new CDecisionTreeClassifier(2, 5); //a decision tree classifier from DecisionTree class
//---
vector open, high, low, close;
int data_size = 1000;
//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1
open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);
high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);
low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);
close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);
matrix X(data_size, 3); //creating the x matrix
//--- Assigning the open, high, and low price values to the x matrix
X.Col(open, 0);
X.Col(high, 1);
X.Col(low, 2);
//--- Since we are using the x variables to predict y, we choose the close price to be the target variable
vector y(data_size);
for (int i=0; i<data_size; i++)
{
if (close[i]>open[i]) //a bullish candle appeared
y[i] = 1; //buy signal
else
{
y[i] = 0; //sell signal
}
}
//--- We split the data into training and testing samples for training and evaluation
matrix X_train, X_test;
vector y_train, y_test;
double train_size = 0.7; //70% of the data should be used for training the rest for testing
int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable
MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples
//--- Normalizing the independent variables
X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether
X_test = scaler.transform(X_test); // we transform the new data this way
//--- Training the model
decision_tree.fit(X_train, y_train); //The training function
//--- Measuring predictive accuracy
vector train_predictions = decision_tree.predict_bin(X_train);
Print("Training results classification report");
Metrics::classification_report(y_train, train_predictions);
//--- Evaluating the model on out-of-sample predictions
vector test_predictions = decision_tree.predict_bin(X_test);
Print("Testing results classification report");
Metrics::classification_report(y_test, test_predictions);
//---
ArraySetAsSeries(rates, true);
return(INIT_SUCCEEDED);
}
//+------------------------------------------------------------------+
//| Expert deinitialization function |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
{
//---
delete (decision_tree); //We have to delete the AI model object from the memory
}
//+------------------------------------------------------------------+
//| Expert tick function |
//+------------------------------------------------------------------+
void OnTick()
{
//--- Making predictions live from the market
CopyRates(Symbol(), PERIOD_D1, 1, 3, rates); //Get the very recent information from the market
vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data
x = scaler.transform(x);
int signal = (int)decision_tree.predict_bin(x);
Comment("Signal = ",signal==1?"BUY":"SELL"); //Ternary operator for checking if the signal is either buy or sell
}
//+------------------------------------------------------------------+
================================================
FILE: Examples/Regressor Model Example.mq5
================================================
//+------------------------------------------------------------------+
//| Regressor Model sample.mq5 |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#property version "1.00"
#include <MALE5\Decision Tree\tree.mqh>
#include <MALE5\preprocessing.mqh>
#include <MALE5\MatrixExtend.mqh> //helper functions for for data manipulations
#include <MALE5\metrics.mqh> //fo measuring the performance
StandardizationScaler scaler; //standardization scaler from preprocessing.mqh
CDecisionTreeRegressor *decision_tree; //a decision tree classifier model
MqlRates rates[];
//+------------------------------------------------------------------+
//| Expert initialization function |
//+------------------------------------------------------------------+
int OnInit()
{
//--- Model selection
decision_tree = new CDecisionTreeRegressor(2, 5); //a decision tree classifier from DecisionTree class
vector open, high, low, close;
int data_size = 1000; //bars
//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1
open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);
high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);
low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);
close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);
matrix X(data_size, 3); //creating the x matrix
//--- Assigning the open, high, and low price values to the x matrix
X.Col(open, 0);
X.Col(high, 1);
X.Col(low, 2);
vector y = close; // The target variable is the close price, using open, high and low values were want to predict the next closing price
//--- We split the data into training and testing samples for training and evaluation
matrix X_train, X_test;
vector y_train, y_test;
double train_size = 0.7; //70% of the data to be used for training the rest 30% for testing
int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable
MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples
//--- Normalizing the independent variables
X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether
X_test = scaler.transform(X_test); // we transform the new data this way
//--- Training the model
decision_tree.fit(X_train, y_train); //The training function
//--- Measuring predictive accuracy
vector train_predictions = decision_tree.predict(X_train);
printf("Decision decision_tree training r2_score = %.3f ",Metrics::RegressionMetric(y_train, train_predictions, METRIC_R_SQUARED));
//--- Evaluating the model on out-of-sample predictions
vector test_predictions = decision_tree.predict(X_test);
printf("Decision decision_tree out-of-sample r2_score = %.3f ",Metrics::r_squared(y_test, test_predictions));
return(INIT_SUCCEEDED);
}
//+------------------------------------------------------------------+
//| Expert deinitialization function |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
{
//---
delete (decision_tree); //We have to delete the AI model object from the memory
}
//+------------------------------------------------------------------+
//| Expert tick function |
//+------------------------------------------------------------------+
void OnTick()
{
//--- Making predictions live from the market
CopyRates(Symbol(), PERIOD_D1, 1, 3, rates); //Get the very recent information from the market
vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data
x = scaler.transform(x);
double predicted_close_price = decision_tree.predict(x);
Comment("Next closing price predicted is = ",predicted_close_price);
}
//+------------------------------------------------------------------+
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2023 Omega Joctan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: MqPlotLib/plots.mqh
================================================
//+------------------------------------------------------------------+
//| plots.mqh |
//| Copyright 2022, Fxalgebra.com |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, Fxalgebra.com"
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
//| defines |
//+------------------------------------------------------------------+
#include <Graphics\Graphic.mqh>
#include <MALE5\MatrixExtend.mqh>
class CPlots
{
protected:
CGraphic *graph;
long m_chart_id;
int m_subwin;
int m_x1, m_x2;
int m_y1, m_y2;
string m_font_family;
bool m_chart_show;
string m_plot_names[];
ENUM_CURVE_TYPE m_curve_type;
bool GraphCreate(string plot_name);
vector m_x, m_y;
string x_label, y_label;
public:
CPlots(long chart_id=0, int sub_win=0 ,int x1=30, int y1=40, int x2=550, int y2=310, string font_family="Consolas", bool chart_show=true);
~CPlots(void);
bool Plot(string plot_name, vector& x, vector& y, string x_axis_label, string y_axis_label, string label, ENUM_CURVE_TYPE curve_type=CURVE_POINTS_AND_LINES,color clr = clrDodgerBlue, bool points_fill = true);
bool AddPlot(vector &v,string label="plt",color clr=clrOrange);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CPlots::CPlots(long chart_id=0, int sub_win=0 ,int x1=30, int y1=40, int x2=550, int y2=310, string font_family="Consolas", bool chart_show=true):
m_chart_id(chart_id),
m_subwin(sub_win),
m_x1(x1),
m_y1(y1),
m_x2(x2),
m_y2(y2),
m_font_family(font_family),
m_chart_show(chart_show)
{
graph = new CGraphic();
ChartRedraw(m_chart_id);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CPlots::~CPlots(void)
{
for (int i=0; i<ArraySize(m_plot_names); i++)
ObjectDelete(m_chart_id,m_plot_names[i]);
delete(graph);
ChartRedraw();
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CPlots::GraphCreate(string plot_name)
{
ChartRedraw(m_chart_id);
ArrayResize(m_plot_names,ArraySize(m_plot_names)+1);
m_plot_names[ArraySize(m_plot_names)-1] = plot_name;
ChartSetInteger(m_chart_id, CHART_SHOW, m_chart_show);
if(!graph.Create(m_chart_id, plot_name, m_subwin, m_x1, m_y1, m_x2, m_y2))
{
printf("Failed to Create graphical object on the Main chart Err = %d", GetLastError());
return(false);
}
return (true);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CPlots::Plot(
string plot_name,
vector& x,
vector& y,
string x_axis_label,
string y_axis_label,
string label,
ENUM_CURVE_TYPE curve_type=CURVE_POINTS_AND_LINES,
color clr = clrDodgerBlue,
bool points_fill = true
)
{
if (!this.GraphCreate(plot_name))
return (false);
//---
this.m_x = x;
this.m_y = y;
this.x_label = x_axis_label;;
this.y_label = y_axis_label;
double x_arr[], y_arr[];
MatrixExtend::VectorToArray(x, x_arr);
MatrixExtend::VectorToArray(y, y_arr);
m_curve_type = curve_type;
graph.CurveAdd(x_arr, y_arr, ColorToARGB(clr), m_curve_type, label);
graph.XAxis().Name(x_axis_label);
graph.XAxis().NameSize(13);
graph.YAxis().Name(y_axis_label);
graph.YAxis().NameSize(13);
graph.FontSet(m_font_family, 13);
graph.CurvePlotAll();
graph.Update();
return(true);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CPlots::AddPlot(vector &v, string label="plt",color clr=clrOrange)
{
double x_arr[], y_arr[];
MatrixExtend::VectorToArray(this.m_x, x_arr);
MatrixExtend::VectorToArray(v, y_arr);
if (!graph.CurveAdd(x_arr, y_arr, ColorToARGB(clr), m_curve_type, label))
{
printf("%s failed to add a plot to the existing plot Err =%d",__FUNCTION__,GetLastError());
return false;
}
graph.XAxis().Name(this.x_label);
graph.XAxis().NameSize(13);
graph.YAxis().Name(this.y_label);
graph.YAxis().NameSize(13);
graph.FontSet(m_font_family, 13);
graph.CurvePlotAll();
graph.Update();
return(true);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Neural Networks/Pattern Nets.mqh
================================================
//+------------------------------------------------------------------+
//| Pattern Nets.mqh |
//| Copyright 2022, Fxalgebra.com |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, Fxalgebra.com"
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
//| Neural network type for pattern recognition/ can be used to |
//| to predict discrete data target variables. They are widely known |
//| as classification Neural Networks |
//+------------------------------------------------------------------+
#include <MALE5\MatrixExtend.mqh>
#include <MALE5\preprocessing.mqh>
#ifndef RANDOM_STATE
#define RANDOM_STATE 42
#endif
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
enum activation
{
AF_HARD_SIGMOID_ = AF_HARD_SIGMOID,
AF_SIGMOID_ = AF_SIGMOID,
AF_SWISH_ = AF_SWISH,
AF_SOFTSIGN_ = AF_SOFTSIGN,
AF_TANH_ = AF_TANH
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CPatternNets
{
private:
vector W_CONFIG;
vector W; //Weights vector
vector B; //Bias vector
activation A_FX;
protected:
ulong inputs;
ulong outputs;
ulong rows;
vector HL_CONFIG;
bool SoftMaxLayer;
vector classes;
void SoftMaxLayerFX(matrix<double> &mat);
public:
CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false);
~CPatternNets(void);
int PatternNetFF(vector &in_vector);
vector PatternNetFF(matrix &xmatrix);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CPatternNets::CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false)
{
A_FX = ActivationFx;
inputs = xmatrix.Cols();
rows = xmatrix.Rows();
SoftMaxLayer = SoftMaxLyr;
//--- Normalize data
if (rows != yvector.Size())
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return;
}
classes = MatrixExtend::Unique(yvector);
outputs = classes.Size();
HL_CONFIG.Copy(HL_NODES);
HL_CONFIG.Resize(HL_CONFIG.Size()+1); //Add the output layer
HL_CONFIG[HL_CONFIG.Size()-1] = (int)outputs; //Append one node to the output layer
//---
W_CONFIG.Resize(HL_CONFIG.Size());
B.Resize((ulong)HL_CONFIG.Sum());
//--- GENERATE WEIGHTS
ulong layer_input = inputs;
for (ulong i=0; i<HL_CONFIG.Size(); i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
layer_input = (ulong)HL_CONFIG[i];
}
W.Resize((ulong)W_CONFIG.Sum());
W = MatrixExtend::Random(0.0, 1.0, (int)W.Size(),RANDOM_STATE); //Gen weights
B = MatrixExtend::Random(0.0,1.0,(int)B.Size(),RANDOM_STATE); //Gen bias
//---
#ifdef DEBUG_MODE
Comment(
"< - - - P A T T E R N N E T S - - - >\n",
"HIDDEN LAYERS + OUTPUT ",HL_CONFIG,"\n",
"INPUTS ",inputs," | OUTPUTS ",outputs," W CONFIG ",W_CONFIG,"\n",
"activation ",EnumToString(A_FX)," SoftMaxLayer = ",bool(SoftMaxLayer)
);
Print("WEIGHTS ",W,"\nBIAS ",B);
#endif
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CPatternNets::~CPatternNets(void)
{
ZeroMemory(W);
ZeroMemory(B);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
int CPatternNets::PatternNetFF(vector &in_vector)
{
matrix L_INPUT = {}, L_OUTPUT={}, L_WEIGHTS = {};
vector v_weights ={};
ulong w_start = 0;
L_INPUT = MatrixExtend::VectorToMatrix(in_vector);
vector L_BIAS_VECTOR = {};
matrix L_BIAS_MATRIX = {};
ulong b_start = 0;
for (ulong i=0; i<W_CONFIG.Size(); i++)
{
MatrixExtend::Copy(W,v_weights,w_start,ulong(W_CONFIG[i]));
L_WEIGHTS = MatrixExtend::VectorToMatrix(v_weights,L_INPUT.Rows());
MatrixExtend::Copy(B,L_BIAS_VECTOR,b_start,(ulong)HL_CONFIG[i]);
L_BIAS_MATRIX = MatrixExtend::VectorToMatrix(L_BIAS_VECTOR);
#ifdef DEBUG_MODE
Print("--> ",i);
Print("L_WEIGHTS\n",L_WEIGHTS,"\nL_INPUT\n",L_INPUT,"\nL_BIAS\n",L_BIAS_MATRIX);
#endif
L_OUTPUT = L_WEIGHTS.MatMul(L_INPUT);
L_OUTPUT = L_OUTPUT+L_BIAS_MATRIX; //Add bias
//---
if (i==W_CONFIG.Size()-1) //Last layer
{
if (SoftMaxLayer)
{
Print("Before softmax\n",L_OUTPUT);
SoftMaxLayerFX(L_OUTPUT);
Print("After\n",L_OUTPUT);
}
else
L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX));
}
else
L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX));
//---
L_INPUT.Copy(L_OUTPUT); //Assign outputs to the inputs
w_start += (ulong)W_CONFIG[i]; //New weights copy
b_start += (ulong)HL_CONFIG[i];
}
#ifdef DEBUG_MODE
Print("--> outputs\n ",L_OUTPUT);
#endif
vector v_out = MatrixExtend::MatrixToVector(L_OUTPUT);
return((int)classes[v_out.ArgMax()]);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CPatternNets::SoftMaxLayerFX(matrix<double> &mat)
{
vector<double> ret = MatrixExtend::MatrixToVector(mat);
ret.Activation(ret, AF_SOFTMAX);
mat = MatrixExtend::VectorToMatrix(ret, mat.Cols());
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
vector CPatternNets::PatternNetFF(matrix &xmatrix)
{
vector v(xmatrix.Rows());
for (ulong i=0; i<xmatrix.Rows(); i++)
v[i] = PatternNetFF(xmatrix.Row(i));
return (v);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Neural Networks/README.md
================================================
## Kohonen Maps (Self-Organizing Maps)
This documentation explains the `CKohonenMaps` class in MQL5, which implements **Kohonen Maps**, also known as **Self-Organizing Maps (SOM)**, for clustering and visualization tasks.
**I. Kohonen Maps Theory:**
Kohonen Maps are a type of **artificial neural network** used for unsupervised learning, specifically for **clustering** and **visualization** of high-dimensional data. They work by:
1. **Initializing a grid of neurons:** Each neuron is associated with a weight vector representing its position in the high-dimensional space.
2. **Iteratively presenting data points:**
* For each data point:
* Find the **winning neuron** (closest neuron in terms of distance) based on the weight vectors.
* Update the weights of the winning neuron and its **neighborhood** towards the data point, with decreasing influence as the distance from the winning neuron increases.
3. **Convergence:** After a certain number of iterations (epochs), the weight vectors of the neurons become organized in a way that reflects the underlying structure of the data.
**II. CKohonenMaps Class:**
The `CKohonenMaps` class provides functionalities for implementing Kohonen Maps in MQL5:
**Public Functions:**
* **CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42)** Constructor, allows setting hyperparameters:
* `clusters`: Number of clusters (default: 2).
* `alpha`: Learning rate (default: 0.01).
* `epochs`: Number of training epochs (default: 100).
* `random_state`: Random seed for reproducibility (default: 42).
* `~CKohonenMaps(void)` Destructor.
* `void fit(const matrix &x)` Trains the model on the provided data (`x`).
* `int predict(const vector &x)` Predicts the cluster label for a single data point (`x`).
* `vector predict(const matrix &x)` Predicts cluster labels for multiple data points (`x`).
**Internal Functions:**
* `Euclidean_distance(const vector &v1, const vector &v2)`: Calculates the Euclidean distance between two vectors.
* `CalcTimeElapsed(double seconds)`: Converts seconds to a human-readable format (not relevant for core functionality).
**III. Additional Notes:**
* The class internally uses the `CPlots` class (not documented here) for potential visualization purposes.
* The `c_matrix` and `w_matrix` member variables store the cluster assignments and weight matrix, respectively.
* Choosing the appropriate number of clusters is crucial for the quality of the results.
By understanding the theoretical foundation and functionalities of the `CKohonenMaps` class, MQL5 users can leverage Kohonen Maps for:
* **Clustering:** Group similar data points together based on their features.
* **Data visualization:** Project the high-dimensional data onto a lower-dimensional space (e.g., a 2D grid of neurons) for easier visualization, potentially using the `CPlots` class.
## Pattern Recognition Neural Network
This documentation explains the `CPatternNets` class in MQL5, which implements a **feed-forward neural network** for pattern recognition tasks.
**I. Neural Network Theory:**
A feed-forward neural network consists of interconnected layers of **neurons**. Each neuron receives input from the previous layer, applies an **activation function** to transform the signal, and outputs the result to the next layer.
**II. CPatternNets Class:**
The `CPatternNets` class provides functionalities for training and using a feed-forward neural network for pattern recognition in MQL5:
**Public Functions:**
* `CPatternNets(matrix &xmatrix, vector &yvector,vector &HL_NODES, activation ActivationFx, bool SoftMaxLyr=false)` Constructor:
* `xmatrix`: Input data matrix (rows: samples, columns: features).
* `yvector`: Target labels vector (corresponding labels for each sample in `xmatrix`).
* `HL_NODES`: Vector specifying the number of neurons in each hidden layer.
* `ActivationFx`: Activation function (enum specifying the type of activation function).
* `SoftMaxLyr`: Flag indicating whether to use a SoftMax layer in the output (default: False).
* `~CPatternNets(void)` Destructor.
* `int PatternNetFF(vector &in_vector)` Performs a forward pass on the network with a single input vector and returns the predicted class label.
* `vector PatternNetFF(matrix &xmatrix)` Performs a forward pass on the network for all rows in the input matrix and returns a vector of predicted class labels.
**Internal Functions:**
* `SoftMaxLayerFX(matrix<double> &mat)`: Applies the SoftMax function to a matrix (used for the output layer if `SoftMaxLyr` is True).
**III. Class Functionality:**
1. **Initialization:**
* The constructor validates data dimensions and parses user-defined hyperparameters.
* The network architecture (number of layers and neurons) is determined based on the provided configuration.
* Weights (connections between neurons) and biases (individual offsets for each neuron) are randomly initialized.
2. **Forward Pass:**
* The provided input vector is fed into the first layer.
* Each layer performs the following steps:
* Calculates the weighted sum of the previous layer's outputs.
* Adds the bias term to the weighted sum.
* Applies the chosen activation function to the result.
* This process continues through all layers until the final output layer is reached.
3. **SoftMax Layer (Optional)**
* If the `SoftMaxLyr` flag is True, the output layer uses the SoftMax function to ensure the output values sum to 1 and represent class probabilities.
4. **Prediction:**
* For single-sample prediction (`PatternNetFF(vector &in_vector)`), the class label with the **highest output value** is returned.
* For batch prediction (`PatternNetFF(matrix &xmatrix)`), a vector containing the predicted class label for each sample in the input matrix is returned.
**IV. Additional Notes:**
* The class provides several debug statements (disabled by default) to print intermediate calculations for debugging purposes.
* The code uses helper functions from the `MatrixExtend` class (not documented here) for matrix and vector operations.
* Choosing the appropriate network architecture, activation function, and learning approach (not implemented in this class) is crucial for optimal performance on specific tasks.
By understanding the theoretical foundation and functionalities of the `CPatternNets` class, MQL5 users can leverage neural networks for various pattern recognition tasks, including:
* **Classification:** Classifying data points into predefined categories based on their features.
* **Anomaly detection:** Identifying data points that deviate significantly from the expected patterns.
* **Feature learning:** Extracting hidden patterns or representations from the data.
## Regression Neural Network
This documentation explains the `CRegressorNets` class in MQL5, which implements a **Multi-Layer Perceptron (MLP)** for regression tasks.
**I. Regression vs. Classification:**
* **Regression:** Predicts continuous output values.
* **Classification:** Assigns data points to predefined categories.
**II. MLP Neural Network:**
An MLP is a type of **feed-forward neural network** used for supervised learning tasks like regression. It consists of:
* **Input layer:** Receives the input data.
* **Hidden layers:** Process and transform the information.
* **Output layer:** Produces the final prediction (continuous value in regression).
**III. CRegressorNets Class:**
The `CRegressorNets` class provides functionalities for training and using an MLP for regression in MQL5:
**Public Functions:**
* `CRegressorNets(vector &HL_NODES, activation ActivationFX=AF_RELU_)` Constructor:
* `HL_NODES`: Vector specifying the number of neurons in each hidden layer.
* `ActivationFX`: Activation function (enum specifying the type of activation function).
* `~CRegressorNets(void)` Destructor.
* `void fit(matrix &x, vector &y)` Trains the model on the provided data (`x` - features, `y` - target values).
* `double predict(vector &x)` Predicts the output value for a single input vector.
* `vector predict(matrix &x)` Predicts output values for all rows in the input matrix.
**Internal Functions (not directly accessible)**
* `CalcTimeElapsed(double seconds)`: Calculates and returns a string representing the elapsed time in a human-readable format (not relevant for core functionality).
* `RegressorNetsBackProp(matrix& x, vector &y, uint epochs, double alpha, loss LossFx=LOSS_MSE_, optimizer OPTIMIZER=OPTIMIZER_ADAM)`: Performs backpropagation for training (details not provided but likely involve calculating gradients and updating weights and biases).
* Optimizer functions (e.g., `AdamOptimizerW`, `AdamOptimizerB`) Implement specific optimization algorithms like Adam for updating weights and biases during training.
**Other Class Members:**
* `mlp_struct mlp`: Stores information about the network architecture (inputs, hidden layers, and outputs).
* `CTensors*` pointers: Represent tensors holding weights, biases, and other internal calculations (specific implementation likely relies on a custom tensor library).
* `matrix` variables: Used for calculations during training and may hold temporary data (e.g., `W_MATRIX`, `B_MATRIX`).
* `vector` variables: Store network configuration details (e.g., `W_CONFIG`, `HL_CONFIG`).
* `bool isBackProp`: Flag indicating if backpropagation is being performed (private).
* `matrix` variables: Used for storing intermediate results during backpropagation (e.g., `ACTIVATIONS`, `Partial_Derivatives`).
**IV. Additional Notes:**
* The class provides various activation function options and supports different loss functions (e.g., Mean Squared Error, Mean Absolute Error) for selecting the appropriate evaluation metric during training.
* The class implements the Adam optimizer, one of several optimization algorithms used for efficient training of neural networks.
* Detailed implementation of the backpropagation algorithm is not provided but is likely the core functionality for training the network.
**Reference**
* [Data Science and Machine Learning (Part 12): Can Self-Training Neural Networks Help You Outsmart the Stock Market?](https://www.mql5.com/en/articles/12209)
* [Data Science and Machine Learning — Neural Network (Part 01): Feed Forward Neural Network demystified](https://www.mql5.com/en/articles/11275)
* [Data Science and Machine Learning — Neural Network (Part 02): Feed forward NN Architectures Design](https://www.mql5.com/en/articles/11334)
================================================
FILE: Neural Networks/Regressor Nets.mqh
================================================
//+------------------------------------------------------------------+
//| neural_nn_lib.mqh |
//| Copyright 2022, Omega Joctan. |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, Omega Joctan."
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
//| Regressor Neural Networks | Neural Networks for solving |
//| regression problems in contrast to classification problems, |
//| here we deal with continuous variables |
//+------------------------------------------------------------------+
#include <MALE5\preprocessing.mqh>;
#include <MALE5\MatrixExtend.mqh>;
#include <MALE5\Metrics.mqh>
#include <MALE5\Tensors.mqh>
#include <MALE5\cross_validation.mqh>
#include "optimizers.mqh"
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
enum activation
{
AF_ELU_ = AF_ELU,
AF_EXP_ = AF_EXP,
AF_GELU_ = AF_GELU,
AF_LINEAR_ = AF_LINEAR,
AF_LRELU_ = AF_LRELU,
AF_RELU_ = AF_RELU,
AF_SELU_ = AF_SELU,
AF_TRELU_ = AF_TRELU,
AF_SOFTPLUS_ = AF_SOFTPLUS
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
enum loss
{
LOSS_MSE_ = LOSS_MSE, // Mean Squared Error
LOSS_MAE_ = LOSS_MAE, // Mean Absolute Error
LOSS_MSLE_ = LOSS_MSLE, // Mean Squared Logarithmic Error
LOSS_POISSON_ = LOSS_POISSON // Poisson Loss
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
struct backprop //This structure returns the loss information obtained from the backpropagation function
{
vector training_loss,
validation_loss;
void Init(ulong epochs)
{
training_loss.Resize(epochs);
validation_loss.Resize(epochs);
}
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
struct mlp_struct //multi layer perceptron information structure
{
ulong inputs;
ulong hidden_layers;
ulong outputs;
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CRegressorNets
{
mlp_struct mlp;
CTensors *Weights_tensor; //Weight Tensor
CTensors *Bias_tensor;
CTensors *Input_tensor;
CTensors *Output_tensor;
protected:
activation A_FX;
loss m_loss_function;
bool trained;
string ConvertTime(double seconds);
//-- for backpropn
vector W_CONFIG;
vector HL_CONFIG;
bool isBackProp;
matrix<double> ACTIVATIONS;
matrix<double> Partial_Derivatives;
int m_random_state;
private:
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual backprop backpropagation(const matrix& x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
public:
CRegressorNets(vector &HL_NODES, activation AF_=AF_RELU_, loss m_loss_function=LOSS_MSE_, int random_state=42);
~CRegressorNets(void);
virtual void fit(const matrix &x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual void fit(const matrix &x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual void fit(const matrix &x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual void fit(const matrix &x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual void fit(const matrix &x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual void fit(const matrix &x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false);
virtual double predict(const vector &x);
virtual vector predict(const matrix &x);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CRegressorNets::CRegressorNets(vector &HL_NODES, activation AF_=AF_RELU_, loss LOSS_=LOSS_MSE_, int random_state=42)
:A_FX(AF_),
m_loss_function(LOSS_),
isBackProp(false),
m_random_state(random_state)
{
HL_CONFIG.Copy(HL_NODES);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CRegressorNets::~CRegressorNets(void)
{
if (CheckPointer(this.Weights_tensor) != POINTER_INVALID) delete(this.Weights_tensor);
if (CheckPointer(this.Bias_tensor) != POINTER_INVALID) delete(this.Bias_tensor);
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CRegressorNets::predict(const vector &x)
{
if (!trained)
{
printf("%s Train the model first before using it to make predictions | call the fit function first",__FUNCTION__);
return 0;
}
matrix L_INPUT = MatrixExtend::VectorToMatrix(x);
matrix L_OUTPUT ={};
for(ulong i=0; i<mlp.hidden_layers; i++)
{
if (isBackProp) //if we are on backpropagation store the inputs to be used for finding derivatives
this.Input_tensor.Add(L_INPUT, i);
L_OUTPUT = this.Weights_tensor.Get(i).MatMul(L_INPUT) + this.Bias_tensor.Get(i); //Weights x INputs + Bias
L_OUTPUT.Activation(L_OUTPUT, ENUM_ACTIVATION_FUNCTION(A_FX)); //Activation
L_INPUT = L_OUTPUT; //Next layer inputs = previous layer outputs
if (isBackProp) this.Output_tensor.Add(L_OUTPUT, i); //Add bias //if we are on backpropagation store the outputs to be used for finding derivatives
}
return(L_OUTPUT[0][0]);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
vector CRegressorNets::predict(const matrix &x)
{
ulong size = x.Rows();
vector v(size);
if (x.Cols() != mlp.inputs)
{
Print("Cen't pass this matrix to a MLP it doesn't have the same number of columns as the inputs given primarily");
return (v);
}
for (ulong i=0; i<size; i++)
v[i] = predict(x.Row(i));
return (v);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
string CRegressorNets::ConvertTime(double seconds)
{
string time_str = "";
uint minutes = 0, hours = 0;
if (seconds >= 60)
{
minutes = (uint)(seconds / 60.0) ;
seconds = fmod(seconds, 1.0) * 60;
time_str = StringFormat("%d Minutes and %.3f Seconds", minutes, seconds);
}
if (minutes >= 60)
{
hours = (uint)(minutes / 60.0);
minutes = minutes % 60;
time_str = StringFormat("%d Hours and %d Minutes", hours, minutes);
}
if (time_str == "")
{
time_str = StringFormat("%.3f Seconds", seconds);
}
return time_str;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerSGD optimizer_weights = optimizer;
OptimizerSGD optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerAdaDelta optimizer_weights = optimizer;
OptimizerAdaDelta optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerAdaGrad optimizer_weights = optimizer;
OptimizerAdaGrad optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerAdam optimizer_weights = optimizer;
OptimizerAdam optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerNadam optimizer_weights = optimizer;
OptimizerNadam optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
backprop CRegressorNets::backpropagation(const matrix& x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
isBackProp = true;
//---
backprop backprop_struct;
backprop_struct.Init(epochs);
ulong rows = x.Rows();
mlp.inputs = x.Cols();
mlp.outputs = 1;
//---
vector v2 = {(double)mlp.outputs}; //Adding the output layer to the mix of hidden layers
HL_CONFIG = MatrixExtend::concatenate(HL_CONFIG, v2);
mlp.hidden_layers = HL_CONFIG.Size();
W_CONFIG.Resize(HL_CONFIG.Size());
//---
if (y.Size() != rows)
{
Print(__FUNCTION__," FATAL | Number of rows in the x matrix is not the same the y vector size ");
return backprop_struct;
}
matrix W, B;
//--- GENERATE WEIGHTS
this.Weights_tensor = new CTensors((uint)mlp.hidden_layers);
this.Bias_tensor = new CTensors((uint)mlp.hidden_layers);
this.Input_tensor = new CTensors((uint)mlp.hidden_layers);
this.Output_tensor = new CTensors((uint)mlp.hidden_layers);
ulong layer_input = mlp.inputs;
for (ulong i=0; i<mlp.hidden_layers; i++)
{
W_CONFIG[i] = layer_input*HL_CONFIG[i];
W = MatrixExtend::Random(0.0, 1.0,(ulong)HL_CONFIG[i],layer_input, m_random_state);
W = W * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Weights_tensor.Add(W, i);
B = MatrixExtend::Random(0.0, 0.5,(ulong)HL_CONFIG[i],1,m_random_state);
B = B * sqrt(2/((double)layer_input + HL_CONFIG[i])); //glorot
this.Bias_tensor.Add(B, i);
layer_input = (ulong)HL_CONFIG[i];
}
//---
if (MQLInfoInteger(MQL_DEBUG))
Comment("<------------------- R E G R E S S O R N E T S ------------------------->\n",
"HL_CONFIG ",HL_CONFIG," TOTAL HL(S) ",mlp.hidden_layers,"\n",
"W_CONFIG ",W_CONFIG," ACTIVATION ",EnumToString(A_FX),"\n",
"NN INPUTS ",mlp.inputs," OUTPUT ",mlp.outputs
);
//--- Optimizer
OptimizerRMSprop optimizer_weights = optimizer;
OptimizerRMSprop optimizer_bias = optimizer;
if (batch_size>0)
{
OptimizerMinBGD optimizer_weights;
OptimizerMinBGD optimizer_bias;
}
//--- Cross validation
CCrossValidation cross_validation;
CTensors *cv_tensor;
matrix validation_data = MatrixExtend::concatenate(x, y);
matrix validation_x;
vector validation_y;
cv_tensor = cross_validation.KFoldCV(validation_data, 10); //k-fold cross validation | 10 folds selected
//---
matrix DELTA = {};
double actual=0, pred=0;
matrix temp_inputs ={};
matrix dB = {}; //Bias Derivatives
matrix dW = {}; //Weight Derivatives
for (ulong epoch=0; epoch<epochs && !IsStopped(); epoch++)
{
double epoch_start = GetTickCount();
uint num_batches = (uint)MathFloor(x.Rows()/(batch_size+DBL_EPSILON));
vector batch_loss(num_batches),
batch_accuracy(num_batches);
vector actual_v(1), pred_v(1), LossGradient = {};
if (batch_size==0) //Stochastic Gradient Descent
{
for (ulong iter=0; iter<rows; iter++) //iterate through all data points
{
pred = predict(x.Row(iter));
actual = y[iter];
pred_v[0] = pred;
actual_v[0] = actual;
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
}
else //Batch Gradient Descent
{
for (uint batch=0, batch_start=0, batch_end=batch_size; batch<num_batches; batch++, batch_start+=batch_size, batch_end=(batch_start+batch_size-1))
{
matrix batch_x = MatrixExtend::Get(x, batch_start, batch_end-1);
vector batch_y = MatrixExtend::Get(y, batch_start, batch_end-1);
rows = batch_x.Rows();
for (ulong iter=0; iter<rows ; iter++) //iterate through all data points
{
pred_v[0] = predict(batch_x.Row(iter));
actual_v[0] = y[iter];
//---
DELTA.Resize(mlp.outputs,1);
for (int layer=(int)mlp.hidden_layers-1; layer>=0 && !IsStopped(); layer--) //Loop through the network backward from last to first layer
{
Partial_Derivatives = this.Output_tensor.Get(int(layer));
temp_inputs = this.Input_tensor.Get(int(layer));
Partial_Derivatives.Derivative(Partial_Derivatives, ENUM_ACTIVATION_FUNCTION(A_FX));
if (mlp.hidden_layers-1 == layer) //Last layer
{
LossGradient = pred_v.LossGradient(actual_v, ENUM_LOSS_FUNCTION(m_loss_function));
DELTA.Col(LossGradient, 0);
}
else
{
W = this.Weights_tensor.Get(layer+1);
DELTA = (W.Transpose().MatMul(DELTA)) * Partial_Derivatives;
}
//-- Observation | DeLTA matrix is same size as the bias matrix
W = this.Weights_tensor.Get(layer);
B = this.Bias_tensor.Get(layer);
//--- Derivatives wrt weights and bias
dB = DELTA;
dW = DELTA.MatMul(temp_inputs.Transpose());
//--- Weights updates
optimizer_weights.update(W, dW);
optimizer_bias.update(B, dB);
this.Weights_tensor.Add(W, layer);
this.Bias_tensor.Add(B, layer);
}
}
pred_v = predict(batch_x);
batch_loss[batch] = pred_v.Loss(batch_y, ENUM_LOSS_FUNCTION(m_loss_function));
batch_loss[batch] = MathIsValidNumber(batch_loss[batch]) ? (batch_loss[batch]>1e6 ? 1e6 : batch_loss[batch]) : 1e6; //Check for nan and return some large value if it is nan
batch_accuracy[batch] = Metrics::r_squared(batch_y, pred_v);
if (show_batch_progress)
printf("----> batch[%d/%d] batch-loss %.5f accuracy %.3f",batch+1,num_batches,batch_loss[batch], batch_accuracy[batch]);
}
}
//--- End of an epoch
vector validation_loss(cv_tensor.SIZE);
vector validation_acc(cv_tensor.SIZE);
for (ulong i=0; i<cv_tensor.SIZE; i++)
{
validation_data = cv_tensor.Get(i);
MatrixExtend::XandYSplitMatrices(validation_data, validation_x, validation_y);
vector val_preds = this.predict(validation_x);;
validation_loss[i] = val_preds.Loss(validation_y, ENUM_LOSS_FUNCTION(m_loss_function));
validation_acc[i] = Metrics::r_squared(validation_y, val_preds);
}
pred_v = this.predict(x);
if (batch_size==0)
{
backprop_struct.training_loss[epoch] = pred_v.Loss(y, ENUM_LOSS_FUNCTION(m_loss_function));
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
else
{
backprop_struct.training_loss[epoch] = batch_loss.Mean();
backprop_struct.training_loss[epoch] = MathIsValidNumber(backprop_struct.training_loss[epoch]) ? (backprop_struct.training_loss[epoch]>1e6 ? 1e6 : backprop_struct.training_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
backprop_struct.validation_loss[epoch] = validation_loss.Mean();
backprop_struct.validation_loss[epoch] = MathIsValidNumber(backprop_struct.validation_loss[epoch]) ? (backprop_struct.validation_loss[epoch]>1e6 ? 1e6 : backprop_struct.validation_loss[epoch]) : 1e6; //Check for nan and return some large value if it is nan
}
double epoch_stop = GetTickCount();
printf("--> Epoch [%d/%d] training -> loss %.8f accuracy %.3f validation -> loss %.5f accuracy %.3f | Elapsed %s ",epoch+1,epochs,backprop_struct.training_loss[epoch],Metrics::r_squared(y, pred_v),backprop_struct.validation_loss[epoch],validation_acc.Mean(),this.ConvertTime((epoch_stop-epoch_start)/1000.0));
}
isBackProp = false;
if (CheckPointer(this.Input_tensor) != POINTER_INVALID) delete(this.Input_tensor);
if (CheckPointer(this.Output_tensor) != POINTER_INVALID) delete(this.Output_tensor);
if (CheckPointer(optimizer)!=POINTER_INVALID)
delete optimizer;
return backprop_struct;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerSGD *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-SGD log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdaDelta *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-AdaDelta log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdaGrad *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-AdaGrad log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerAdam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-Adam log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerNadam *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-Nadam log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CRegressorNets::fit(const matrix &x, const vector &y, OptimizerRMSprop *optimizer, const uint epochs, uint batch_size=0, bool show_batch_progress=false)
{
trained = true; //The fit method has been called
vector epochs_vector(epochs); for (uint i=0; i<epochs; i++) epochs_vector[i] = i+1;
backprop backprop_struct;
backprop_struct = this.backpropagation(x, y, optimizer, epochs, batch_size, show_batch_progress); //Run backpropagation
CPlots plt;
backprop_struct.training_loss = log10(backprop_struct.training_loss); //Logarithmic scalling
plt.Plot("Loss vs Epochs",epochs_vector,backprop_struct.training_loss,"epochs","optimizer-RMSProp log10(loss)","training-loss",CURVE_LINES);
backprop_struct.validation_loss = log10(backprop_struct.validation_loss);
plt.AddPlot(backprop_struct.validation_loss,"validation-loss",clrRed);
while (MessageBox("Close or Cancel Loss Vs Epoch plot to proceed","Training progress",MB_OK)<0)
Sleep(1);
isBackProp = false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Neural Networks/initializers.mqh
================================================
//+------------------------------------------------------------------+
//| initiallizers.mqh |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#include <MALE5\MatrixExtend.mqh>
enum enum_weight_initializers
{
WEIGHT_INTIALIZER_XAVIER, //Xavier weights
WEIGHT_INTIALIZER_HE, //He weights
WEIGHT_INTIALIZER_RANDOM //Random weights
};
//+------------------------------------------------------------------+
//| defines |
//+------------------------------------------------------------------+
class CWeightsInitializers
{
protected:
static double random() //Generates a random float between 0 (inclusive) and 1 (exclusive)
{
return 0 + double((MathRand() / 32767.0) * (0.9 - 0));
}
static double uniform(double low, double high)
{
return low + (high - low) * random();
}
static matrix uniform(double low, double high, ulong rows, ulong cols)
{
matrix return_matrix(rows, cols);
for (ulong i=0; i<rows; i++)
for (ulong j=0; j<cols; j++)
return_matrix[i][j] = uniform(low, high);
return return_matrix;
}
static vector uniform(double low, double high, ulong size)
{
vector v(size);
for (ulong i=0; i<size; i++)
v[i] = uniform(low, high);
return v;
}
public:
CWeightsInitializers(void);
~CWeightsInitializers(void);
static matrix Xavier(const ulong inputs, const ulong outputs); //Xavier or Glorot initialization
static matrix He(const ulong inputs, const ulong outputs);
static matrix Random(const ulong inputs, const ulong outputs);
static matrix Initializer(const ulong inputs, const ulong outputs, enum_weight_initializers WEIGHT_INIT=WEIGHT_INTIALIZER_XAVIER);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix CWeightsInitializers::Xavier(const ulong inputs,const ulong outputs)
{
#ifdef RANDOM_STATE
MathSrand(RANDOM_STATE);
#endif
double limit = sqrt(6/(inputs+outputs+DBL_EPSILON));
return uniform(-limit, limit, inputs, outputs);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix CWeightsInitializers::He(const ulong inputs, const ulong outputs)
{
#ifdef RANDOM_STATE
MathSrand(RANDOM_STATE);
#endif
double limit = sqrt(2 / (inputs+DBL_EPSILON));
matrix W(inputs, outputs);
for (ulong i=0; i<outputs; i++)
{
vector v = uniform(-limit, limit, inputs);
W.Col(v, i);
}
return uniform(-limit, limit, inputs, outputs);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix CWeightsInitializers::Random(const ulong inputs,const ulong outputs)
{
#ifdef RANDOM_STATE
MathSrand(RANDOM_STATE);
#endif
matrix W(inputs, outputs);
for (ulong i=0; i<inputs; i++)
for (ulong j=0; j<outputs; j++)
W[i][j] = random();
return W;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix CWeightsInitializers::Initializer(const ulong inputs,const ulong outputs,enum_weight_initializers WEIGHT_INIT=WEIGHT_INTIALIZER_XAVIER)
{
matrix W = {};
switch(WEIGHT_INIT)
{
case WEIGHT_INTIALIZER_HE:
W = He(inputs, outputs);
break;
case WEIGHT_INTIALIZER_RANDOM:
W = Random(inputs, outputs);
break;
case WEIGHT_INTIALIZER_XAVIER:
W = Xavier(inputs, outputs);
break;
}
return W;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Neural Networks/kohonen maps.mqh
================================================
//+------------------------------------------------------------------+
//| kohonen maps.mqh |
//| Copyright 2022, Fxalgebra.com |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, Fxalgebra.com"
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include <MALE5\MatrixExtend.mqh>
#include <MALE5\Tensors.mqh>
#include <MALE5\preprocessing.mqh>
#include <MALE5\MqPlotLib\plots.mqh>
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CKohonenMaps
{
protected:
C2DTensor *cluster_tensor;
CPlots plt;
double Euclidean_distance(const vector &v1, const vector &v2);
string CalcTimeElapsed(double seconds);
matrix c_matrix; //Clusters
matrix w_matrix; //Weights matrix
vector w_vector; //weights vector
matrix o_matrix; //Output layer matrix
uint m_clusters;
double m_alpha;
uint m_epochs;
int m_random_state;
ulong n, m;
public:
CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42);
~CKohonenMaps(void);
void fit(const matrix &x);
int predict(const vector &x);
vector predict(const matrix &x);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CKohonenMaps::CKohonenMaps(uint clusters=2, double alpha=0.01, uint epochs=100, int random_state=42)
:m(clusters),
m_alpha(alpha),
m_epochs(epochs),
m_random_state(random_state),
m_clusters(clusters)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CKohonenMaps::fit(const matrix &x)
{
n = (uint)x.Cols(); //number of features
ulong rows = x.Rows();
cluster_tensor = new C2DTensor();
cluster_tensor.Init((int)m);
w_matrix =MatrixExtend::Random(0.0, 1.0, n, m, m_random_state);
#ifdef DEBUG_MODE
Print("w x\n",w_matrix,"\nMatrix\n",x);
#endif
vector D(m); //Euclidean distance btn clusters
for (uint epoch=0; epoch<m_epochs; epoch++)
{
double epoch_start = GetMicrosecondCount()/(double)1e6, epoch_stop=0;
for (ulong i=0; i<rows; i++)
{
for (ulong j=0; j<m; j++)
{
D[j] = Euclidean_distance(x.Row(i),w_matrix.Col(j));
}
#ifdef DEBUG_MODE
//Print("Euc distance ",D," Winning cluster ",D.ArgMin());
#endif
//--- weights update
ulong min = D.ArgMin();
if (epoch == m_epochs-1) //last iteration
cluster_tensor[(int)min].Vector = x.Row(i);;
vector w_new = w_matrix.Col(min) + (m_alpha * (x.Row(i) - w_matrix.Col(min)));
w_matrix.Col(w_new, min);
}
epoch_stop =GetMicrosecondCount()/(double)1e6;
printf("Epoch [%d/%d] | %sElapsed ",epoch+1,m_epochs, CalcTimeElapsed(epoch_stop-epoch_start));
} //end of the training
//---
#ifdef DEBUG_MODE
Print("\nNew weights\n",w_matrix);
#endif
//---
vector v;
matrix plotmatrix(rows, m);
for (uint i=0; i<this.cluster_tensor.Size(); i++)
{
v = this.cluster_tensor[i].Vector;
plotmatrix.Col(v, i);
}
vector x_axis(plotmatrix.Rows()); for (ulong i=0; i<x_axis.Size(); i++) x_axis[i] = (int)i+1;
CColorGenerator clr;
plt.Plot("kom", x_axis, plotmatrix.Col(0), "map", "clusters","cluster"+string(1),CURVE_POINTS,clr.Next()); //plot the first cluster
for (ulong i=1; i<plotmatrix.Cols(); i++) //start at the second column in the matrix | the second cluster
{
plt.AddPlot(plotmatrix.Col(i), "cluster"+string(i+1),clr.Next()); //Add the rest of clusters to the existing plot
}
//---
if (MQLInfoInteger(MQL_DEBUG))
{
Print("\nclusters");
cluster_tensor.Print_();
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CKohonenMaps::~CKohonenMaps(void)
{
ZeroMemory(c_matrix);
ZeroMemory(w_matrix);
ZeroMemory(w_vector);
ZeroMemory(o_matrix);
delete (cluster_tensor);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CKohonenMaps:: Euclidean_distance(const vector &v1, const vector &v2)
{
double dist = 0;
if(v1.Size() != v2.Size())
Print(__FUNCTION__, " v1 and v2 not matching in size");
else
{
double c = 0;
for(ulong i=0; i<v1.Size(); i++)
c += MathPow(v1[i] - v2[i], 2);
dist = MathSqrt(c);
}
return(dist);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
int CKohonenMaps::predict(const vector &v)
{
if (n != v.Size())
{
printf("%s Can't predict the cluster | the input vector size is not the same as the trained matrix cols",__FUNCTION__);
return(-1);
}
vector D(m); //Euclidean distance btn clusters
for (ulong j=0; j<m; j++)
D[j] = Euclidean_distance(v, w_matrix.Col(j));
return((int)D.ArgMin());
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
vector CKohonenMaps::predict(const matrix &x)
{
vector v(x.Rows());
for (ulong i=0; i<x.Rows(); i++)
v[i] = predict(x.Row(i));
return(v);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
string CKohonenMaps::CalcTimeElapsed(double seconds)
{
string time_str = "";
uint minutes=0, hours=0;
if (seconds >= 60)
time_str = StringFormat("%d Minutes and %.3f Seconds ",minutes=(int)round(seconds/60.0), ((int)seconds % 60));
if (minutes >= 60)
time_str = StringFormat("%d Hours %d Minutes and %.3f Seconds ",hours=(int)round(minutes/60.0), minutes, ((int)seconds % 60));
else
time_str = StringFormat("%.3f Seconds ",seconds);
return time_str;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Neural Networks/optimizers.mqh
================================================
//+------------------------------------------------------------------+
//| optimizers.mqh |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
//|Class containing optimizers for updating neural network parameters|
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
//| |
//| Adam (Adaptive Moment Estimation): Adam is an adaptive learning |
//| rate optimizer that maintains learning rates for each parameter |
//| and adapts them based on the first and second moments of the |
//| gradients. It's known for its fast convergence and robustness to |
//| noisy gradients |
//| |
//+------------------------------------------------------------------+
/*enum Optimizer {
OPTIMIZER_SGD, //Stochastic Gradient Descent
OPTIMIZER_MiniBatchGD, //Mini-Batch Gradiend Descent
OPTIMIZER_Adam, //Adaptive Moment Estimation
OPTIMIZER_RMSprop, //Root Mean Square Propagation
OPTIMIZER_Adagrad, //Adaptive Gradient Descent
OPTIMIZER_Adadelta, //Adadelta
OPTIMIZER_Nadam //Nesterov-accelerated Adaptive Moment Estimation
};
*/
class OptimizerAdam
{
protected:
int time_step;
double m_learning_rate;
double m_beta1;
double m_beta2;
double m_epsilon;
matrix moment; //first moment estimate
matrix cache; //second moment estimate
public:
OptimizerAdam(double learning_rate=0.01, double beta1=0.9, double beta2=0.999, double epsilon=1e-8);
~OptimizerAdam(void);
virtual void update(matrix ¶meters, matrix &gradients);
};
//+------------------------------------------------------------------+
//| Initializes the Adam optimizer with hyperparameters. |
//| |
//| learning_rate: Step size for parameter updates |
//| beta1: Decay rate for the first moment estimate |
//| (moving average of gradients). |
//| beta2: Decay rate for the second moment estimate |
//| (moving average of squared gradients). |
//| epsilon: Small value for numerical stability. |
//+------------------------------------------------------------------+
OptimizerAdam::OptimizerAdam(double learning_rate=0.010000, double beta1=0.9, double beta2=0.999, double epsilon=1e-8):
time_step(0),
m_learning_rate(learning_rate),
m_beta1(beta1),
m_beta2(beta2),
m_epsilon(epsilon)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerAdam::~OptimizerAdam(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerAdam::update(matrix ¶meters,matrix &gradients)
{
// Initialize moment and cache matrices if not already initialized
if (moment.Rows() != parameters.Rows() || moment.Cols() != parameters.Cols())
{
moment.Resize(parameters.Rows(), parameters.Cols());
moment.Fill(0.0);
}
if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())
{
cache.Resize(parameters.Rows(), parameters.Cols());
cache.Fill(0.0);
}
this.time_step++;
this.moment = this.m_beta1 * this.moment + (1 - this.m_beta1) * gradients;
this.cache = this.m_beta2 * this.cache + (1 - this.m_beta2) * MathPow(gradients, 2);
//--- Bias correction
matrix moment_hat = this.moment / (1 - MathPow(this.m_beta1, this.time_step));
matrix cache_hat = this.cache / (1 - MathPow(this.m_beta2, this.time_step));
parameters -= (this.m_learning_rate * moment_hat) / (MathPow(cache_hat, 0.5) + this.m_epsilon);
}
//+------------------------------------------------------------------+
//| |
//| |
//| Stochastic Gradient Descent (SGD): |
//| |
//| This is the simplest optimizer |
//| It updates the parameters using the gradients of the loss |
//| function computed on individual training samples or mini-batches.|
//| |
//| |
//+------------------------------------------------------------------+
class OptimizerSGD
{
protected:
double m_learning_rate;
public:
OptimizerSGD(double learning_rate=0.01);
~OptimizerSGD(void);
virtual void update(matrix ¶meters, matrix &gradients);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerSGD::OptimizerSGD(double learning_rate=0.01):
m_learning_rate(learning_rate)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerSGD::~OptimizerSGD(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerSGD::update(matrix ¶meters, matrix &gradients)
{
parameters -= this.m_learning_rate * gradients;
}
//+------------------------------------------------------------------+
//| Batch Gradient Descent (BGD): This optimizer computes the |
//| gradients of the loss function on the entire training dataset |
//| and updates the parameters accordingly. It can be slow and |
//| memory-intensive for large datasets but tends to provide a |
//| stable convergence. |
//+------------------------------------------------------------------+
class OptimizerMinBGD: public OptimizerSGD
{
public:
OptimizerMinBGD(double learning_rate=0.01);
~OptimizerMinBGD(void);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerMinBGD::OptimizerMinBGD(double learning_rate=0.010000): OptimizerSGD(learning_rate)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerMinBGD::~OptimizerMinBGD(void)
{
}
//+------------------------------------------------------------------+
//| |
//| |
//| |
//| RMSprop (Root Mean Square Propagation): RMSprop is similar to |
//| Adam but only uses the first moment of the gradients to adapt the|
//| moment learning rates. It's effective in handling non-stationary |
//| objectives and can converge quickly for many problems. |
//| |
//| |
//+------------------------------------------------------------------+
class OptimizerRMSprop
{
protected:
double m_learning_rate;
double m_decay_rate;
double m_epsilon;
matrix<double> cache;
//Dividing double/matrix causes compilation error | this is the fix to the issue
matrix divide(const double numerator, const matrix &denominator)
{
matrix res = denominator;
for (ulong i=0; i<denominator.Rows(); i++)
res.Row(numerator / denominator.Row(i), i);
return res;
}
public:
OptimizerRMSprop(double learning_rate=0.01, double decay_rate=0.9, double epsilon=1e-8);
~OptimizerRMSprop(void);
virtual void update(matrix& parameters, matrix& gradients);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerRMSprop::OptimizerRMSprop(double learning_rate=0.01, double decay_rate=0.9, double epsilon=1e-8):
m_learning_rate(learning_rate),
m_decay_rate(decay_rate),
m_epsilon(epsilon)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerRMSprop::~OptimizerRMSprop(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerRMSprop::update(matrix ¶meters,matrix &gradients)
{
if (cache.Rows()!=parameters.Rows() || cache.Cols()!=parameters.Cols())
{
cache.Init(parameters.Rows(), parameters.Cols());
cache.Fill(0.0);
}
//---
cache += m_decay_rate * cache + (1 - m_decay_rate) * MathPow(gradients, 2);
parameters -= divide(m_learning_rate, cache + m_epsilon) * gradients;
}
//+------------------------------------------------------------------+
//| |
//| |
//| |
//| Adagrad (Adaptive Gradient Algorithm): |
//| |
//| Adagrad adapts the learning rates for each parameter based on the|
//| historical gradients. It performs larger updates for infrequent |
//| parameters and smaller updates for frequent parameters, making |
//| it suitable for sparse data. |
//| |
//| |
//| |
//+------------------------------------------------------------------+
class OptimizerAdaGrad
{
protected:
double m_learning_rate;
double m_epsilon;
matrix cache;
//Dividing double/matrix causes compilation error | this is the fix to the issue
matrix divide(const double numerator, const matrix &denominator)
{
matrix res = denominator;
for (ulong i=0; i<denominator.Rows(); i++)
res.Row(numerator / denominator.Row(i), i);
return res;
}
public:
OptimizerAdaGrad(double learning_rate=0.01, double epsilon=1e-8);
~OptimizerAdaGrad(void);
virtual void update(matrix ¶meters, matrix &gradients);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerAdaGrad::OptimizerAdaGrad(double learning_rate=0.01,double epsilon=1e-8):
m_learning_rate(learning_rate),
m_epsilon(epsilon)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerAdaGrad::~OptimizerAdaGrad(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerAdaGrad::update(matrix ¶meters,matrix &gradients)
{
if (cache.Rows()!=parameters.Rows() || cache.Cols()!=parameters.Cols())
{
cache.Resize(parameters.Rows(), parameters.Cols());
cache.Fill(0.0);
}
//---
cache += MathPow(gradients, 2);
parameters -= divide(this.m_learning_rate, MathSqrt(cache + this.m_epsilon)) * gradients;
}
//+------------------------------------------------------------------+
//| |
//| Adadelta: |
//| |
//| Adadelta is an extension of Adagrad that aims to address |
//| its tendency to decrease the learning rate over time. It uses a |
//| more sophisticated update rule that accounts for the accumulated |
//| gradients over a window of time. |
//| |
//+------------------------------------------------------------------+
class OptimizerAdaDelta
{
protected:
double m_decay_rate;
double m_epsilon, m_gamma, lr;
matrix cache; //Exponential moving average of squared gradients
public:
OptimizerAdaDelta(double learning_rate=0.01, double decay_rate=0.95, double gamma=0.9, double epsilon=1e-8);
~OptimizerAdaDelta(void);
virtual void update(matrix ¶meters, matrix &gradients);
};
//+------------------------------------------------------------------+
//| decay_rate: Decay rate for the EMA of squared deltas |
//| epsilons: Smoothing term to avoid division by zero |
//| gamma: Momentum coefficient (hyperparameter) |
//+------------------------------------------------------------------+
OptimizerAdaDelta::OptimizerAdaDelta(double learning_rate=0.01, double decay_rate=0.95, double gamma=0.9, double epsilon=1e-8):
m_decay_rate(decay_rate),
m_epsilon(epsilon),
m_gamma(gamma),
lr(learning_rate)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerAdaDelta::~OptimizerAdaDelta(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerAdaDelta::update(matrix ¶meters, matrix &gradients)
{
// Initialize moment and cache matrices if not already initialized
if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())
{
cache.Resize(parameters.Rows(), parameters.Cols());
cache.Fill(0.0);
}
//---
this.cache = m_decay_rate * this.cache + (1 - m_decay_rate) * MathPow(gradients, 2);
matrix delta = lr * sqrt(this.cache + m_epsilon) / sqrt(pow(gradients, 2) + m_epsilon); //Adaptive learning rate
matrix momentum_term = this.m_gamma * parameters + (1 - this.m_gamma) * gradients;
parameters -= delta * momentum_term;
}
//+------------------------------------------------------------------+
//| |
//| |
//| Nadam (Nesterov-accelerated Adaptive Moment Estimation): |
//| |
//| Nadam combines Nesterov momentum with Adam optimizer, which |
//| results in faster convergence and better generalization. |
//| |
//| |
//+------------------------------------------------------------------+
class OptimizerNadam: protected OptimizerAdam
{
protected:
double m_gamma;
public:
OptimizerNadam(double learning_rate=0.01, double beta1=0.9, double beta2=0.999, double gamma=0.9, double epsilon=1e-8);
~OptimizerNadam(void);
virtual void update(matrix ¶meters, matrix &gradients);
};
//+------------------------------------------------------------------+
//| Initializes the Adam optimizer with hyperparameters. |
//| |
//| learning_rate: Step size for parameter updates |
//| beta1: Decay rate for the first moment estimate |
//| (moving average of gradients). |
//| beta2: Decay rate for the second moment estimate |
//| (moving average of squared gradients). |
//| epsilon: Small value for numerical stability. |
//+------------------------------------------------------------------+
OptimizerNadam::OptimizerNadam(double learning_rate=0.010000, double beta1=0.9, double beta2=0.999, double gamma=0.9, double epsilon=1e-8)
:OptimizerAdam(learning_rate, beta1, beta2, epsilon),
m_gamma(gamma)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
OptimizerNadam::~OptimizerNadam(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void OptimizerNadam::update(matrix ¶meters, matrix &gradients)
{
// Initialize moment and cache matrices if not already initialized
if (moment.Rows() != parameters.Rows() || moment.Cols() != parameters.Cols())
{
moment.Resize(parameters.Rows(), parameters.Cols());
moment.Fill(0.0);
}
if (cache.Rows() != parameters.Rows() || cache.Cols() != parameters.Cols())
{
cache.Resize(parameters.Rows(), parameters.Cols());
cache.Fill(0.0);
}
this.time_step++;
// Update moment and cache similar to Adam
moment = m_beta1 * moment + (1 - m_beta1) * gradients;
cache = m_beta2 * cache + (1 - m_beta2) * MathPow(gradients, 2);
// Bias correction
matrix moment_hat = moment / (1 - MathPow(m_beta1, time_step));
matrix cache_hat = cache / (1 - MathPow(m_beta2, time_step));
// Nesterov accelerated gradient
matrix nesterov_moment = m_gamma * moment_hat + (1 - m_gamma) * gradients;
// Update parameters
parameters -= m_learning_rate * nesterov_moment / sqrt(cache_hat + m_epsilon);
}
================================================
FILE: README.md
================================================
<p align="center">
<img width="25%" align="center" src="https://github.com/MegaJoctan/MALE5/assets/65341461/5a903238-921d-4f09-8e27-1847d4052af3" alt="logo">
</p>
<h1 align="center">
M A L E 5
</h1>
<p align="center">
A python-like Machine Learning Library for MQL5
</p>
<p align="center">
<a href="https://github.com/MegaJoctan/MALE5/releases" target="_blank">
<img src="https://img.shields.io/github/v/release/MegaJoctan/MALE5?color=%2334D058&label=Version" alt="Version">
</a>
<a href="https://github.com/MegaJoctan/MALE5/stargazers">
<img src="https://img.shields.io/github/stars/MegaJoctan/MALE5?color=brightgreen&label=Stars" alt="Stars"/>
</a>
<a href="https://github.com/MegaJoctan/MALE5/blob/main/LICENSE">
<img src="https://img.shields.io/github/license/MegaJoctan/MALE5?color=blue" alt="License"/>
</a>
<a>
<img src="https://img.shields.io/badge/Platform-Win32%20|%20Linux%20|%20macOS-blue?color=blue" alt="Platform Win32 | Linux | macOS"/>
</a>
</p>
<p align="center">
<a href="https://discord.gg/2qgcadfgrx" style="text-decoration:none">
<img src="https://img.shields.io/badge/Discord-%237289DA?style=flat&logo=discord"/>
</a>
<a href="https://t.me/fxalgebra_discussion" style="text-decoration:none">
<img src="https://img.shields.io/badge/Telegram-%232CA5E0?style=flat&logo=telegram"/>
</a>
</p>
<p align="center">
English | <a href="README_russian.md">Russian</a>
</p>
## About the Project
MALE5 is a machine-learning repository for creating trading systems in the c++ like, MQL5 programming language.
It was developed to help build machine learning-based trading robots, effortlessly in the [MetaTrader5](https://www.metatrader5.com/en/automated-trading/metaeditor) platform
**My goal is to make the library**
- **Python-Like, Simple to use**
- **Flexible:** To be usable within a Trading Robot(EA), Custom Indicator, and Scripts.
- **Resource-efficient:** To make it not consume a lot of resources and memory.
**Disclaimer**
*This project is not for MQL5 programming language beginners. Those with prior experience of machine learning using python might find this project easy to understand*
## Installing
Download the zip file from the releases section extract the library. Go under MQL5 folder in your MetaEditor, from there paste the MALE5 directory you extracted under the Include folder.
## Getting Started with the Library
In this project, machine learning models can be placed into three categories. *See the tables below*.
## 01: Predictive Models
Unlike others which are mostly used for analysis and data processing. These AI models when given an inputs (predictors) in matrices or vectors they provide predictions. Currently Available models include.
| **Model Type** | **Models** |
|------------------------------|----------------------------------|
| **Linear Models** | - Linear regression |
| | - Logistic regression |
| | - Ridge regression |
| | - Polynomial regression |
| **Decision Trees** | - Decision tree |
| **Ensemble Models** | - AdaBoost tree |
| | - Random forest |
| **Support Vector Machine** | - SVM |
| **Neural Networks** | - Regressor Neural Networks |
| | - Pattern Recognition Neural Networks |
| | - Kohonen Maps |
| **Naïve Bayes** | - Naïve Bayes models |
### Traininng the predictive models
All the predictive models functions for training and deploying them for predictions follow similar patterns.
[](https://www.youtube.com/watch?v=wKk85PZ2ra8&t=1s)
#### For Regression Problems
A regression problem is a type of predictive modeling problem where the goal is to predict a continuous output variable based on one or more input features. In other words, given input data, the model aims to determine a continuous value. For example, predicting the next closing price of a financial instrument.
**How Regression Models Work**
Firstly, we import the necessary libraries we need. In this example, we will be using the Decision Tree Regressor.
```MQL5
#include <MALE5\Decision Tree\tree.mqh>
#include <MALE5\preprocessing.mqh>
#include <MALE5\MatrixExtend.mqh> //helper functions for data manipulations
#include <MALE5\metrics.mqh> //for measuring the performance
StandardizationScaler scaler; //standardization scaler from preprocessing.mqh
CDecisionTreeRegressor *decision_tree; //a decision tree regressor model
```
1. **Data Collection**
Gather a dataset with input features (also called predictors or attributes) and the corresponding target variable (the output).
```MQL5
vector open, high, low, close;
int data_size = 1000;
//--- Getting the open, high, low, and close values for the past 1000 bars, starting from the recent closed bar of 1
open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);
high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);
low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);
close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);
matrix X(data_size, 3); //creating the X matrix
//--- Assigning the open, high, and low price values to the X matrix
X.Col(open, 0);
X.Col(high, 1);
X.Col(low, 2);
vector y = close; // The target variable is the close price
```
2. **Preprocessing**
This involves cleaning and preprocessing the data, which might involve handling missing values, normalizing the data, and encoding categorical variables if possible in the data.
```MQL5
//--- We split the data into training and testing samples for training and evaluation
matrix X_train, X_test;
vector y_train, y_test;
double train_size = 0.7; //70% of the data should be used for training, the rest for testing
int random_state = 42; //we put a random state to shuffle the data
MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the X and y data into training and testing samples
//--- Normalizing the independent variables
X_train = scaler.fit_transform(X_train); // fit the scaler on the training data and transform the data
X_test = scaler.transform(X_test); // transform the test data
// Print the processed data for verification
//Print("X_train\n",X_train,"\nX_test\n",X_test,"\ny_train\n",y_train,"\ny_test\n",y_test);
```
3. **Model Selection**
Choose a regression algorithm. Common algorithms for this type of problem include linear regression, decision trees, support vector regression (SVR), k-nearest neighbors (K-NN), and neural networks.
```MQL5
//--- Model selection
decision_tree = new CDecisionTreeRegressor(2, 5); //a decision tree regressor from DecisionTree class
```
4. **Training**
Use the training data to teach the model how to map input features to the correct continuous output values. During training, the model learns patterns in the data.
```MQL5
//--- Training the model
decision_tree.fit(X_train, y_train); // The training function
```
5. **Evaluation**
Assess the model's performance on a separate test dataset to ensure it can generalize well to new, unseen data. Metrics like R-squared (R²) score, mean absolute error (MAE), and root mean squared error (RMSE) are often used to evaluate the model.
```MQL5
//--- Measuring predictive accuracy
vector train_predictions = decision_tree.predict(X_train);
printf("Decision Tree training R² score = %.3f ", Metrics::r_squared(y_train, train_predictions));
//--- Evaluating the model on out-of-sample predictions
vector test_predictions = decision_tree.predict(X_test);
printf("Decision Tree out-of-sample R² score = %.3f ", Metrics::r_squared(y_test, test_predictions));
```
6. **Prediction**
Once trained and evaluated, the model can be used to predict the continuous values of new, unseen data points.
```MQL5
void OnTick()
{
//--- Making predictions live from the market
CopyRates(Symbol(), PERIOD_D1, 1, 1, rates); // Get the very recent information from the market
vector x = {rates[0].open, rates[0].high, rates[0].low}; // Assigning data from the recent candle in a similar way to the training data
double predicted_close_price = decision_tree.predict(x);
Comment("Next closing price predicted is = ", predicted_close_price);
}
```
**Types of Regression**
- **Simple Linear Regression:** Predicting a continuous target variable based on a single input feature.
- **Multiple Linear Regression:** Predicting a continuous target variable based on multiple input features.
- **Polynomial Regression:** Predicting a continuous target variable using polynomial terms of the input features.
- **Non-Linear Regression:** Predicting a continuous target variable using non-linear relationships between input features and the target variable.
#### For Classification Problems
A classification problem is a type of predictive modeling problem where the goal is to predict the category or class that a given data point belongs to. In other words, given input data, the model aims to determine which of the predefined classes the input belongs to. Forexample when trying to predict whether the next candle will be either bullish or bearish.
**How Classification Models Work**
Firstly, we import the necessary libraries we need. In this example we will be using the Decision Tree Classifier.
```MQL5
#include <MALE5\Decision Tree\tree.mqh>
#include <MALE5\preprocessing.mqh>
#include <MALE5\MatrixExtend.mqh> //helper functions for data manipulations
#include <MALE5\metrics.mqh> //fo measuring the performance
StandardizationScaler scaler; //standardization scaler from preprocessing.mqh
CDecisionTreeClassifier *decision_tree; //a decision tree classifier model
```
1. **Data Collection**
Gather a dataset with input features (also called predictors or attributes) and the corresponding class labels (the output).
>
```MQL5
vector open, high, low, close;
int data_size = 1000;
//--- Getting the open, high, low and close values for the past 1000 bars, starting from the recent closed bar of 1
open.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_OPEN, 1, data_size);
high.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_HIGH, 1, data_size);
low.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_LOW, 1, data_size);
close.CopyRates(Symbol(), PERIOD_D1, COPY_RATES_CLOSE, 1, data_size);
decision_tree = new CDecisionTreeClassifier(2, 5);
matrix X(data_size, 3); //creating the x matrix
//--- Assigning the open, high, and low price values to the x matrix
X.Col(open, 0);
X.Col(high, 1);
X.Col(low, 2);
```
2. **Preprocessing**
This involves clearning and preprocess the data, which might involve analytical techniques such as handling missing values, **normalizing the data**, and encoding categorical variables.
```MQL5
//--- We split the data into training and testing samples for training and evaluation
matrix X_train, X_test;
vector y_train, y_test;
double train_size = 0.7; //70% of the data should be used for training the rest for testing
int random_state = 42; //we put a random state to shuffle the data so that a machine learning model understands the patterns and not the order of the dataset, this makes the model durable
MatrixExtend::TrainTestSplitMatrices(X, y, X_train, y_train, X_test, y_test, train_size, random_state); // we split the x and y data into training and testing samples
//--- Normalizing the independent variables
X_train = scaler.fit_transform(X_train); // we fit the scaler on the training data and transform the data alltogether
X_test = scaler.transform(X_test); // we transform the new data this way
//Print("X_train\n",X_train,"\nX_test\n",X_test,"\ny_train\n",y_train,"\ny_test\n",y_test);
```
3. **Model Selection**
Choose a classification algorithm. Common algorithms for this type of problem include; logistic regression, decision trees, support vector machines (SVM), k-nearest neighbors (K-NN), and neural networks.
```MQL5
//--- Model selection
decision_tree = new CDecisionTreeClassifier(2, 5); //a decision tree classifier from DecisionTree class
```
4. **Training**
Use the training data to teach the model how to map input features to the correct class labels. During training, the model learns patterns in the data.
```MQL5
//--- Training the model
decision_tree.fit(X_train, y_train); //The training function
```
5. **Evaluation**
Assess the model's performance on a separate test dataset to ensure it can generalize well to new, unseen data. Metrics like accuracy score, precision, recall, and F1 score are often used to evaluate the model.
```MQL5
//--- Measuring predictive accuracy
vector train_predictions = decision_tree.predict_bin(X_train);
printf("Decision decision_tree training accuracy = %.3f ",Metrics::accuracy_score(y_train, train_predictions));
//--- Evaluating the model on out-of-sample predictions
vector test_predictions = decision_tree.predict_bin(X_test);
printf("Decision decision_tree out-of-sample accuracy = %.3f ",Metrics::accuracy_score(y_test, test_predictions));
```
6. **Prediction**
Once trained and evaluated, the model can be used to predict the class labels of new, unseen data points.
Unlike the `predict` function which is used to obtain predictions in regressors. The predictive functions for the classifiers have different slightly for different name starting with the word predict.
- **predict_bin** - This function can be used to predict the classes in as integer values for classification models. For example: The next candle will be ***bullish*** or ***bearish***
- **predict_proba** - This function predicts the probabilities of a certain classes as double values from 0 for a 0% probability chance to 1 for a 100% probability chance. For example: [0.64, 0.36] probability the next candle will be bullish is 64%, while the probability the next candle will be bearish is 36%
```
void OnTick()
{
//--- Making predictions live from the market
CopyRates(Symbol(), PERIOD_D1, 1, 1, rates); //Get the very recent information from the market
vector x = {rates[0].open, rates[0].high, rates[0].low}; //Assigning data from the recent candle in a similar way to the training data
double predicted_close_price = decision_tree.predict_bin(x);
Comment("Next closing price predicted is = ",predicted_close_price);
}
```
**Types of Classification**
- **Binary Classification:** There are only two classes. For example, classifying emails as spam or not spam.
- **Multiclass Classification:** There are more than two classes. For example, classifying types of animals in an image (e.g., cats, dogs, birds).
- **Multilabel Classification:** Each instance can be assigned multiple classes. For example, tagging a photo with multiple labels like "beach", "sunset", and "vacation".
## 02: Clustering Algorithms
These algorithms are for the special purppose of classifying and grouping data with similar patterns and contents together, they excel in data mining situations
| **Model Type** | **Models** |
|------------------------------|----------------------------------|
| **Nearest Neighbors** | - KNN nearest neighbors |
| **K-Means Clustering** | - k-Means clustering algorithm |
| **Neural Networks** | - Kohonen Maps |
## 03: Dimensionality Reduction Algorithms
These algorithms are widely used in situations where the dataset is huge and needs adjustmets. they excel at reducing the size of datasets without losing much information. Consider them as the algorithms to zip and shread information in the proper way.
| **Model Type** | **Models** |
|----------------------------------------|-------------------------------------|
| **Dimensionality Reduction Algorithms**| - Linear Discriminant Analysis (LDA)|
| | - Non Negative Matrix Factorization (NMF) |
| | - Principal Component Analysis (PCA) |
| | - Truncated SVD |
## Basic Library functionality & helpers
* [MatrixExtend (MatrixExtend.mqh)](https://github.com/MegaJoctan/MALE5/wiki#matrixextendmatrixextendmqh)
* [Cross Validation Library (cross_validation.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Cross-Validation-Library)
* [Linear Algebra Library (linalg.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Linear-Algebra-Library)
* [Kernels library (kernels.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Kernels-Library)
* [Metrics library (metrics.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Metrics-library)
* [Pre-processing library (preprocessing.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Pre-processing-library)
* [Tensor library (Tensor.mqh)](https://github.com/MegaJoctan/MALE5/wiki/Tensor-Library)
## Opening an issue
You can also post bug reports and feature requests (only) in [GitHub issues](https://github.com/MegaJoctan/MALE5/issues).
## Support the Project
If you find this project helpful, Support us by taking one or more of the actions
[BuyMeCoffee](https://www.buymeacoffee.com/omegajoctan)
[OurProducts](https://www.mql5.com/en/users/omegajoctan/seller)
Register to our recommended broker:
[ICMarkets](https://icmarkets.com/?camp=74639)
## Let's work together
[HIRE ME on MQL5.com by clicking on this link](https://www.mql5.com/en/job/new?prefered=omegajoctan)
================================================
FILE: Sklearn/Cluster/DBSCAN.mqh
================================================
//+------------------------------------------------------------------+
//| DBSCAN.mqh |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#include <MALE5\MatrixExtend.mqh>
#include <MALE5\linalg.mqh>
//+------------------------------------------------------------------+
//| defines |
//+------------------------------------------------------------------+
class CDBSCAN
{
protected:
double m_epsilon;
uint m_minsamples;
vector labels;
vector get_neighbors(matrix &x, ulong point_index);
bool expand_cluster(matrix &x, ulong point_index, ulong cluster_id, const vector &neighbors, const vector &cluster_labels);
public:
CDBSCAN(double epsilon, uint min_samples);
~CDBSCAN(void);
vector fit_predict(matrix &x);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CDBSCAN::CDBSCAN(double epsilon, uint min_samples):
m_epsilon(epsilon),
m_minsamples(min_samples)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CDBSCAN::~CDBSCAN(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
vector CDBSCAN::get_neighbors(matrix &x, ulong point_index)
{
vector neighbors = {};
for (ulong i=0, count=0; i<x.Rows(); i++)
{
if (LinAlg::norm(x.Row(point_index), x.Row(i)) < this.m_epsilon && i != point_index)
{
count++;
neighbors.Resize(count);
neighbors[count-1] = (int)i; //Append the neighbor
}
}
return neighbors;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CDBSCAN::expand_cluster(matrix &x, ulong point_index, ulong cluster_id, const vector &neighbors, const vector &cluster_labels)
{
this.labels[point_index] = (int)cluster_id;
vector points_to_explore = {(int)point_index};
Print("points to explore in a cluster: ",points_to_explore);
while (points_to_explore.Size()>0)
{
vector current_point = {points_to_explore[0]};
points_to_explore.Resize(0);
Print("current point: ", current_point);
vector current_neighbors = this.get_neighbors(x, (int)current_point[0]);
if (current_neighbors.Size() >= this.m_minsamples)
for (int neighbor_index=0; neighbor_index<(int)current_neighbors.Size(); neighbor_index++)
{
if (this.labels[neighbor_index] == 0 || this.labels[neighbor_index] == -1)
{
if (this.labels[neighbor_index] == 0)
{
points_to_explore.Resize(1);
points_to_explore[0] = neighbor_index;
Print("points to explore: ",points_to_explore);
}
this.labels[neighbor_index] = (int)cluster_id;
}
}
}
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
vector CDBSCAN::fit_predict(matrix &x)
{
this.labels.Resize(x.Rows()); labels.Fill(0);
ulong cluster_id = 0;
for (ulong i=0; i<x.Rows(); i++)
{
if (this.labels[i] != 0)
continue;
vector neighbors = get_neighbors(x, i);
if (neighbors.Size() < this.m_minsamples)
this.labels[i] = -1; //Mark as noise
else
{
cluster_id++;
Print("Expand cluster: ",i);
expand_cluster(x, i, cluster_id, neighbors, this.labels);
}
}
return this.labels;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Sklearn/Cluster/Hierachical Clustering.mqh
================================================
//+------------------------------------------------------------------+
//| Hierachical Clustering.mqh |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#include "Base.mqh"
//+------------------------------------------------------------------+
//| defines |
//+------------------------------------------------------------------+
enum linkage_enum
{
LINKAGE_SINGLE,
LINKAGE_COMPLETE,
LINKAGE_AVG
};
class CAgglomerativeClustering
{
protected:
linkage_enum linkage;
vector labels;
matrix clusters_keys;
matrix calc_distance_matrix(matrix &x, vector &cluster_members);
public:
CAgglomerativeClustering(linkage_enum linkage_type=LINKAGE_SINGLE);
~CAgglomerativeClustering(void);
void fit(matrix &x);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CAgglomerativeClustering::CAgglomerativeClustering(linkage_enum linkage_type=LINKAGE_SINGLE)
:linkage(linkage_type)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CAgglomerativeClustering::~CAgglomerativeClustering(void)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix CAgglomerativeClustering::calc_distance_matrix(matrix &x, vector &cluster_members)
{
clusters_keys.Init(1, x.Cols()); //initializes the clusters_keys such that each data point is initially in its own cluster
for (ulong i=0; i<x.Cols(); i++) clusters_keys.Col(clusters_keys.Col(i).Fill(i), i); //Filll the initial clusters_keys matrix with their columns incremental values
matrix distance(x.Rows(), x.Rows());
distance.Fill(0.0);
vector v1, v2;
vector ith_element, jth_element;
for (ulong i=0; i<distance.Cols(); i++)
{
ith_element = cluster_members[(int)clusters_keys[i]];
for (ulong j=0; j<distance.Cols(); j++)
{
jth_element = cluster_members[(int)clusters_keys[j]];
v1 = x.Col(i); v2 = x.Col(j);
distance[i][j] = Base::norm(v1, v2);
}
}
return distance;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CAgglomerativeClustering::fit(matrix &x)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Sklearn/Cluster/KMeans.mqh
================================================
//+------------------------------------------------------------------+
//| KMeans.mqh |
//| Copyright 2022, Omega Joctan. |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, Omega Joctan."
#property link "https://www.mql5.com/en/users/omegajoctan"
//+------------------------------------------------------------------+
#include <Graphics\Graphic.mqh>
CGraphic graph;
#include <MALE5\MatrixExtend.mqh>
//+------------------------------------------------------------------+
enum errors
{
KM_ERR001, //clusters not matching in size Error
KM_ERR002
};
//+------------------------------------------------------------------+
class CKMeans
{
private:
ulong n; //number of samples
uint m_clusters;
ulong m_cols;
matrix InitialCentroids;
vector<double> cluster_assign;
protected:
matrix Matrix;
bool ErrMsg(errors err);
bool ScatterCurvePlots(
string obj_name,
double &x[],
double &y[],
double &curveArr[],
string legend,
string x_axis_label = "x-axis",
string y_axis_label = "y-axis",
color clr = clrDodgerBlue,
bool points_fill = true
);
public:
CKMeans(const matrix &_Matrix, int clusters=3);
~CKMeans(void);
void KMeansClustering(matrix &clustered_matrix, matrix ¢roids, int iterations = 1, bool rand_cluster =false);
void ElbowMethod(const int initial_k=1, int total_k=10, bool showPlot = true);
void FilterZero(vector &vec);
void matrixtoArray(matrix &mat, double &Array[]);
};
//+------------------------------------------------------------------+
CKMeans::CKMeans(const matrix &_Matrix, int clusters=3)
{
m_clusters = clusters;
Matrix.Copy(_Matrix);
m_cols = Matrix.Cols();
n = Matrix.Rows(); //number of elements | Matrix Rows
}
//+------------------------------------------------------------------+
CKMeans::~CKMeans(void)
{
ZeroMemory(m_clusters);
ZeroMemory(InitialCentroids);
ZeroMemory(cluster_assign);
}
//+------------------------------------------------------------------+
void CKMeans::KMeansClustering(matrix &clustered_matrix, matrix ¢roids, int iterations = 1, bool rand_cluster =false)
{
InitialCentroids.Resize(m_clusters, m_cols);
cluster_assign.Resize(n);
clustered_matrix.Resize(m_clusters, m_cols*n);
clustered_matrix.Fill(NULL);
vector cluster_comb_v = {};
matrix cluster_comb_m = {};
vector rand_v = {};
ulong rand_ = 0;
for(ulong i=0; i<m_clusters; i++)
{
rand_ = rand_cluster ? (ulong)MathFloor(i*(n/m_clusters)) : i;
rand_v = Matrix.Row(rand_);
InitialCentroids.Row(rand_v, i);
}
//---
vector v_row;
matrix rect_distance = {}; //matrix to store rectilinear distances
rect_distance.Reshape(n, m_clusters);
vector v_matrix = {}, v_centroid = {};
double output = 0;
//---
for(int iter=0; iter<iterations; iter++)
{
for(ulong i=0; i<rect_distance.Rows(); i++)
for(ulong j=0; j<rect_distance.Cols(); j++)
{
v_matrix = Matrix.Row(i);
v_centroid = InitialCentroids.Row(j);
ZeroMemory(output);
for(ulong k=0; k<v_matrix.Size(); k++)
output += MathAbs(v_matrix[k] - v_centroid[k]); //Rectilinear distance
rect_distance[i][j] = output;
}
//--- Assigning the Clusters
matrix cluster_cent = {}; //cluster centroids
ulong cluster = 0;
for(ulong i=0; i<rect_distance.Rows(); i++)
{
v_row = rect_distance.Row(i);
cluster = v_row.ArgMin();
cluster_assign[i] = (double)cluster;
}
vector temp_cluster_assign = cluster_assign;
//--- Combining the clusters
for(ulong i=0, index =0; i<cluster_assign.Size(); i++)
{
ZeroMemory(cluster_cent);
bool classified = false;
for(ulong j=0, count = 0; j<temp_cluster_assign.Size(); j++)
{
if(cluster_assign[i] == temp_cluster_assign[j])
{
classified = true;
count++;
cluster_comb_m.Resize(count, m_cols);
cluster_comb_m.Row(Matrix.Row(j), count-1);
cluster_cent.Resize(count, m_cols);
// New centroids
cluster_cent.Row(Matrix.Row(j), count-1);
temp_cluster_assign[j] = -100; //modify the cluster item so it can no longer be found
}
else
continue;
}
//---
if(classified == true)
{
// solving for new cluster and updtating the old ones
cluster_comb_v = MatrixExtend::MatrixToVector(cluster_comb_m);
if(iter == iterations-1)
clustered_matrix.Row(cluster_comb_v, index);
index++;
//---
vector x_y_z = {0, 0};
ZeroMemory(rand_v);
for(ulong k=0; k<cluster_cent.Cols(); k++)
{
x_y_z.Resize(cluster_cent.Cols());
rand_v = cluster_cent.Col(k);
x_y_z[k] = rand_v.Mean();
}
InitialCentroids.Row(x_y_z, i);
}
if(index >= m_clusters)
break;
}
} //end of iterations
ZeroMemory(centroids);
centroids.Copy(InitialCentroids);
}
//+------------------------------------------------------------------+
bool CKMeans::ErrMsg(errors err)
{
switch(err)
{
case KM_ERR001:
printf("%s Clusters not matching in Size ", EnumToString(KM_ERR001));
break;
default:
break;
}
return(true);
}
//+------------------------------------------------------------------+
void CKMeans::ElbowMethod(const int initial_k=1, int total_k=10, bool showPlot = true)
{
matrix clustered_mat, _centroids = {};
if(total_k > (int)n)
total_k = (int)n; //k should always be less than n
vector centroid_v= {}, x_y_z= {};
vector short_v = {}; //vector for each point
vector minus_v = {}; //vector to store the minus operation output
double wcss = 0;
double WCSS[];
ArrayResize(WCSS, total_k);
double kArray[];
ArrayResize(kArray, total_k);
for(int k=initial_k, count_k=0; k<ArraySize(WCSS)+initial_k; k++, count_k++)
{
wcss = 0;
m_clusters = k;
KMeansClustering(clustered_mat, _centroids);
for(ulong i=0; i<_centroids.Rows(); i++)
{
centroid_v = _centroids.Row(i);
x_y_z = clustered_mat.Row(i);
FilterZero(x_y_z);
for(ulong j=0; j<x_y_z.Size()/m_cols; j++)
{
MatrixExtend::Copy(x_y_z, short_v, uint(j*m_cols), (uint)m_cols);
//--- WCSS ( within cluster sum of squared residuals )
minus_v = (short_v - centroid_v);
minus_v = MathPow(minus_v, 2);
wcss += minus_v.Sum();
}
}
WCSS[count_k] = wcss;
kArray[count_k] = k;
}
Print("WCSS");
ArrayPrint(WCSS);
Print("kArray");
ArrayPrint(kArray);
//--- Plotting the Elbow on the graph
if(showPlot)
{
ObjectDelete(0, "elbow");
ScatterCurvePlots("elbow", kArray, WCSS, WCSS, "Elbow line", "k", "WCSS");
}
}
//+------------------------------------------------------------------+
void CKMeans::FilterZero(vector &vec)
{
vector new_vec = {};
for(ulong i=0, count =0; i<vec.Size(); i++)
{
if(vec[i] != NULL)
{
count++;
new_vec.Resize(count);
new_vec[count-1] = vec[i];
}
else
continue;
}
vec.Copy(new_vec);
}
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
bool CKMeans::ScatterCurvePlots(
string obj_name,
double &x[],
double &y[],
double &curveArr[],
string legend,
string x_axis_label = "x-axis",
string y_axis_label = "y-axis",
color clr = clrDodgerBlue,
bool points_fill = true
)
{
if(!graph.Create(0, obj_name, 0, 30, 70, 800, 640))
{
printf("Failed to Create graphical object on the Main chart Err = %d", GetLastError());
return(false);
}
ChartSetInteger(0, CHART_SHOW, true);
//---
//graph.CurveAdd(x,y,clrBlack,CURVE_POINTS,y_axis_label);
graph.CurveAdd(x, curveArr, clr, CURVE_POINTS_AND_LINES, legend);
graph.XAxis().Name(x_axis_label);
graph.XAxis().NameSize(13);
graph.YAxis().Name(y_axis_label);
graph.YAxis().NameSize(13);
graph.FontSet("Lucida Console", 13);
graph.CurvePlotAll();
graph.Update();
return(true);
}
//+------------------------------------------------------------------+
void CKMeans::matrixtoArray(matrix &mat, double &Array[])
{
ArrayFree(Array);
ArrayResize(Array, int(mat.Rows()*mat.Cols()));
int index = 0;
for(ulong i=0; i<mat.Rows(); i++)
for(ulong j=0; j<mat.Cols(); j++, index++)
{
Array[index] = mat[i][j];
}
}
//+------------------------------------------------------------------+
================================================
FILE: Sklearn/Cluster/base.mqh
================================================
//+------------------------------------------------------------------+
//| BaseClustering.mqh |
//| Copyright 2023, Omega Joctan |
//| https://www.mql5.com/en/users/omegajoctan |
//+------------------------------------------------------------------+
#property copyright "Copyright 2023, Omega Joctan"
#property link "https://www.mql5.com/en/users/omegajoctan"
#include <MALE5\MatrixExtend.mqh>
#include <MALE5\linalg.mqh>
#include <MALE5\MqPlotLib\plots.mqh>
//+------------------------------------------------------------------+
//| defines |
//+------------------------------------------------------------------+
class BaseClustering
{
public:
BaseClustering(void);
~BaseClustering(void);
static matrix Get(matrix &X, vector &index, int axis=0);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
matrix BaseClustering::Get(matrix &X, vector &index, int axis=0)
{
matrix ret_matrix = {};
ulong row_col=0;
bool isRows = true;
switch(axis)
{
case 0:
row_col = X.Rows();
isRows = true;
break;
case 1:
row_col = X.Cols();
isRows = false;
break;
default:
printf("%s Invalid axis %d ",__FUNCTION__,axis);
return ret_matrix;
break;
}
//---
ret_matrix.Resize(isRows?index.Size():X.Rows(), isRows?X.Cols(): index.Size());
for (ulong i=0, count=0; i<row_col; i++)
for (ulong j=0; j<index.Size(); j++)
{
if (isRows)
{
if (i==index[j])
{
if (isRows)
ret_matrix.Row(X.Row(i), count);
else
ret_matrix.Col(X.Col(i), count);
count++;
}
}
}
return ret_matrix;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
================================================
FILE: Sklearn/Decomposition/LDA.mqh
================================================
//+------------
gitextract_ly_vy9wi/ ├── .github/ │ └── FUNDING.yml ├── .gitignore ├── Examples/ │ ├── Classifier Model Example.mq5 │ └── Regressor Model Example.mq5 ├── LICENSE ├── MqPlotLib/ │ └── plots.mqh ├── Neural Networks/ │ ├── Pattern Nets.mqh │ ├── README.md │ ├── Regressor Nets.mqh │ ├── initializers.mqh │ ├── kohonen maps.mqh │ └── optimizers.mqh ├── Numpy/ │ └── Numpy.mqh ├── Pandas/ │ ├── Incremental LE.mqh │ └── pandas.mqh ├── README.md ├── Sklearn/ │ ├── Cluster/ │ │ ├── DBSCAN.mqh │ │ ├── Hierachical Clustering.mqh │ │ ├── KMeans.mqh │ │ └── base.mqh │ ├── Decomposition/ │ │ ├── LDA.mqh │ │ ├── NMF.mqh │ │ ├── PCA.mqh │ │ ├── README.md │ │ ├── TruncatedSVD.mqh │ │ └── base.mqh │ ├── Ensemble/ │ │ ├── AdaBoost.mqh │ │ ├── README.md │ │ └── Random Forest.mqh │ ├── Linear Models/ │ │ ├── Linear Regression.mqh │ │ ├── Logistic Regression.mqh │ │ ├── README.md │ │ └── Ridge.mqh │ ├── Naive Bayes/ │ │ ├── Naive Bayes.mqh │ │ ├── README.md │ │ └── naive bayes visuals.py │ ├── Neighbors/ │ │ └── KNN_nearest_neighbors.mqh │ ├── Tree/ │ │ ├── README.md │ │ └── tree.mqh │ ├── metrics.mqh │ └── preprocessing.mqh ├── Stats Models/ │ ├── ADF.mqh │ ├── ARIMA.mqh │ └── OLS.mqh ├── Tensors.mqh ├── Utils.mqh └── requirements.txt
Condensed preview — 47 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (501K chars).
[
{
"path": ".github/FUNDING.yml",
"chars": 181,
"preview": "# These are supported funding model platforms\n\nko_fi: omegajoctan\n\ncustom: ['https://www.mql5.com/en/users/omegajoctan/s"
},
{
"path": ".gitignore",
"chars": 85,
"preview": "*.ex5\n*.psd\n*.zip\n*.rar\n\n*.xlsx\n\nTodo's.txt\nlogisticwiki.txt\n\n/venv\n\n/Neural Nets Pro"
},
{
"path": "Examples/Classifier Model Example.mq5",
"chars": 5047,
"preview": "//+------------------------------------------------------------------+\n//| Classifi"
},
{
"path": "Examples/Regressor Model Example.mq5",
"chars": 4745,
"preview": "//+------------------------------------------------------------------+\n//| Regress"
},
{
"path": "LICENSE",
"chars": 1069,
"preview": "MIT License\n\nCopyright (c) 2023 Omega Joctan\n\nPermission is hereby granted, free of charge, to any person obtaining a co"
},
{
"path": "MqPlotLib/plots.mqh",
"chars": 5565,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Neural Networks/Pattern Nets.mqh",
"chars": 7577,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Neural Networks/README.md",
"chars": 10577,
"preview": "## Kohonen Maps (Self-Organizing Maps)\n\nThis documentation explains the `CKohonenMaps` class in MQL5, which implements *"
},
{
"path": "Neural Networks/Regressor Nets.mqh",
"chars": 91186,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Neural Networks/initializers.mqh",
"chars": 4793,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Neural Networks/kohonen maps.mqh",
"chars": 7764,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Neural Networks/optimizers.mqh",
"chars": 20075,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "README.md",
"chars": 18224,
"preview": "<p align=\"center\">\n <img width=\"25%\" align=\"center\" src=\"https://github.com/MegaJoctan/MALE5/assets/65341461/5a903238-9"
},
{
"path": "Sklearn/Cluster/DBSCAN.mqh",
"chars": 4924,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Cluster/Hierachical Clustering.mqh",
"chars": 3413,
"preview": "//+------------------------------------------------------------------+\n//| Hierach"
},
{
"path": "Sklearn/Cluster/KMeans.mqh",
"chars": 9925,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Cluster/base.mqh",
"chars": 2442,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Decomposition/LDA.mqh",
"chars": 10156,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Decomposition/NMF.mqh",
"chars": 5958,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Decomposition/PCA.mqh",
"chars": 10472,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Decomposition/README.md",
"chars": 11297,
"preview": "## Linear Discriminant Analysis (LDA) \n\nThis documentation explains the `CLDA` class in MQL5, which implements **Linear "
},
{
"path": "Sklearn/Decomposition/TruncatedSVD.mqh",
"chars": 5734,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Decomposition/base.mqh",
"chars": 5126,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Ensemble/AdaBoost.mqh",
"chars": 11785,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Ensemble/README.md",
"chars": 7005,
"preview": "## AdaBoost Ensemble Learning\n\nThis explanation covers the concept of AdaBoost and its implementation in `adaboost.mqh` "
},
{
"path": "Sklearn/Ensemble/Random Forest.mqh",
"chars": 13034,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Linear Models/Logistic Regression.mqh",
"chars": 6558,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Linear Models/README.md",
"chars": 9442,
"preview": "## Theory Overview: Linear Regression\n\nLinear regression is a statistical method for modeling the relationship between a"
},
{
"path": "Sklearn/Linear Models/Ridge.mqh",
"chars": 2821,
"preview": "//+------------------------------------------------------------------+\n//| R"
},
{
"path": "Sklearn/Naive Bayes/Naive Bayes.mqh",
"chars": 14716,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/Naive Bayes/README.md",
"chars": 2806,
"preview": "## Naive Bayes Classifier\n\nThis documentation explains the `CNaiveBayes` class in MQL5, which implements a **Naive Bayes"
},
{
"path": "Sklearn/Naive Bayes/naive bayes visuals.py",
"chars": 462,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Tue Feb 14 10:37:36 2023\n\n@author: Omega Joctan\n\"\"\"\n\nimport pandas as pd\nimport m"
},
{
"path": "Sklearn/Neighbors/KNN_nearest_neighbors.mqh",
"chars": 12160,
"preview": "//+------------------------------------------------------------------+\n//| KNN_ne"
},
{
"path": "Sklearn/Tree/README.md",
"chars": 3368,
"preview": "## Decision Trees in MQL5: Classification and Regression\n\nDecision trees are powerful machine learning algorithms that u"
},
{
"path": "Sklearn/Tree/tree.mqh",
"chars": 23448,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Sklearn/metrics.mqh",
"chars": 14576,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Stats Models/ADF.mqh",
"chars": 30459,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Stats Models/ARIMA.mqh",
"chars": 13662,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Stats Models/OLS.mqh",
"chars": 13224,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Tensors.mqh",
"chars": 9855,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "Utils.mqh",
"chars": 48401,
"preview": "//+------------------------------------------------------------------+\n//| "
},
{
"path": "requirements.txt",
"chars": 227,
"preview": "contourpy==1.0.7\ncycler==0.11.0\nfonttools==4.38.0\nkiwisolver==1.4.4\nmatplotlib==3.7.0\nnumpy==1.24.2\npackaging==23.0\npand"
}
]
// ... and 5 more files (download for full content)
About this extraction
This page contains the full source code of the MegaJoctan/MALE5 GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 47 files (473.0 KB), approximately 103.2k tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.