Repository: gekkowarez/gekkoga
Branch: stable
Commit: d2ebde921333
Files: 6
Total size: 17.4 KB

Directory structure:
gitextract_164nxiq4/

├── .gitignore
├── README.md
├── index.js
├── package.json
├── results/
│   └── .gitkeep
└── run.js

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
node_modules
package-lock.json
results/config
config
!config/sample-config.js


================================================
FILE: README.md
================================================
# gekkoga
Genetic Algorithm for solving optimization of trading strategies using Gekko
## Installation
    node >= 7.x.x required!
1) git clone https://github.com/gekkowarez/gekkoga.git
2) cd gekkoga
3) npm install
4) cp config/sample-config.js config/your-config.js
5) modify your-config.js make sure you have data for currency/asset pair and the daterange
6) node run.js -c config/your-config.js
#### tmux usage:
1) sudo apt-get install tmux
2) tmux new -s gekkoga
3) go to the web folder in your gekko installation (gekko/web)
4) node –max-old-space-size=8192 server.js (for macos: --max_old_space_size=8192)
5) hold CTRL and press b, then press % (to split the screen)
6) goto your gekkoga clone directory
7) node run.js -c config/your-config.js 
8) hold CTRL and press b, then hit d (to detach and run in the background)
9) tmux attach -t gekkoga (to reattach and bring gekkoga to foreground)


================================================
FILE: index.js
================================================
const async = require('async');
const nodemailer = require('nodemailer');
const randomExt = require('random-ext');
const rp = require('request-promise');
const { some } = require('bluebird');
const fs = require('fs-extra');
const flat = require('flat');
const util = require('util');

class Ga {

  constructor({ gekkoConfig, stratName, mainObjective, populationAmt, parallelqueries, minSharpe, variation, mutateElements, notifications, getProperties, apiUrl }, configName ) {
    this.configName = configName.replace(/\.js|config\//gi, "");
    this.stratName = stratName;
    this.mainObjective = mainObjective;
    this.getProperties = getProperties;
    this.apiUrl = apiUrl;
    this.sendemail = notifications.email.enabled;
    this.senderservice = notifications.email.senderservice;
    this.sender = notifications.email.sender;
    this.senderpass = notifications.email.senderpass;
    this.receiver = notifications.email.receiver;
    this.currency = gekkoConfig.watch.currency;
    this.asset = gekkoConfig.watch.asset;
    this.previousBestParams = null;
    this.populationAmt = populationAmt;
    this.parallelqueries = parallelqueries;
    this.minSharpe = minSharpe;
    this.variation = variation;
    this.mutateElements = mutateElements;
    this.baseConfig = {
      watch: gekkoConfig.watch,
      paperTrader: {
        slippage: gekkoConfig.slippage,
        feeTaker: gekkoConfig.feeTaker,
        feeMaker: gekkoConfig.feeMaker,
        feeUsing: gekkoConfig.feeUsing,
        simulationBalance: gekkoConfig.simulationBalance,
        reportRoundtrips: true,
        enabled: true
      },
      writer: {
        enabled: false,
        logpath: ''
      },
      tradingAdvisor: {
        enabled: true,
        method: this.stratName,
      },
      trader: {
        enabled: false,
      },
      backtest: {
        daterange: gekkoConfig.daterange
      },
      backtestResultExporter: {
        enabled: true,
        writeToDisk: false,
        data: {
          stratUpdates: false,
          roundtrips: false,
          stratCandles: true,
          stratCandleProps: [
              'close',
              'start'
          ],
          trades: false
        }
      },
      performanceAnalyzer: {
        riskFreeReturn: 5,
        enabled: true
      },
      valid: true
    };


  }

  // Checks for, and if present loads old .json parameters
  async loadBreakPoint() {

    const fileName = `./results/${this.configName}-${this.currency}_${this.asset}.json`;
    const exists = fs.existsSync(fileName);

    if(exists){

      console.log('Previous config found, loading...');
      return fs.readFile(fileName, 'utf8').then(JSON.parse);

    }

    return false;

  }

  // Allows queued execution via Promise
  queue(items, parallel, ftc) {

    const queued = [];

    return Promise.all(items.map((item) => {

      const mustComplete = Math.max(0, queued.length - parallel + 1);
      const exec = some(queued, mustComplete).then(() => ftc(item));
      queued.push(exec);

      return exec;

    }));

  }

  // Creates a random gene if prop='all', creates one random property otherwise
  createGene(prop) {
    // Is first generation, and previous props available, load them as a start-point
    if (this.previousBestParams === null || this.runstarted) {
      let properties = flat.flatten(this.getProperties());
      return prop === 'all' ? flat.unflatten(properties) : properties[prop];
    } else if ( this.previousBestParams.parameters && !this.runstarted) {
      this.runstarted = 1;
      let properties = flat.flatten(this.previousBestParams.parameters);
      return prop === 'all' ? flat.unflatten(properties) : properties[prop];
    } else {
      throw Error('Could not resolve a suitable state for previousBestParams');
    }
  }

  // Creates random population from genes
  createPopulation() {
    let population = [];

    for (let i = 0; i < this.populationAmt; i++) {

      population.push(this.createGene('all'));

    }

    return population;
  }

  // Pairs two parents returning two new childs
  crossover(a, b) {

    let len = Object.keys(a).length;
    let crossPoint = randomExt.integer(len - 1, 1);
    let tmpA = {};
    let tmpB = {};
    let currPoint = 0;

    for (let i in a) {

      if (a.hasOwnProperty(i) && b.hasOwnProperty(i)) {

        if (currPoint < crossPoint) {

          tmpA[i] = a[i];
          tmpB[i] = b[i];

        } else {

          tmpA[i] = b[i];
          tmpB[i] = a[i];

        }

      }

      currPoint++;

    }

    return [tmpA, tmpB];
  }

  // Mutates object a at most maxAmount times
  mutate(a, maxAmount) {

    let amt = randomExt.integer(maxAmount, 0);
    // flatten, mutate, return unflattened object
    let flattened = flat.flatten(a);
    let allProps = Object.keys(flattened);

    for (let i = 0; i < amt; i++) {
      let position = randomExt.integer(Object.keys(allProps).length - 1, 0);
      let prop = allProps[position];
      flattened[prop] = this.createGene(prop);
    }

    return flat.unflatten(flattened);
  }

  // For the given population and fitness, returns new population and max score
  runEpoch(population, populationProfits, populationSharpes, populationScores) {
    let selectionProb = [];
    let fitnessSum = 0;
    let maxFitness = [0, 0, 0, 0];

    for (let i = 0; i < this.populationAmt; i++) {

     if (this.mainObjective == 'score') {

       if (populationProfits[i] < 0 && populationSharpes[i] < 0) {

         populationScores[i] = (populationProfits[i] * populationSharpes[i]) * -1;

       } else {

         populationScores[i] = Math.tanh(populationProfits[i] / 3) * Math.tanh(populationSharpes[i] / 0.25);

       }

       if (populationScores[i] > maxFitness[2]) {

         maxFitness = [populationProfits[i], populationSharpes[i], populationScores[i], i];

       }

     } else if (this.mainObjective == 'profit') {

        if (populationProfits[i] > maxFitness[0]) {

          maxFitness = [populationProfits[i], populationSharpes[i], populationScores[i], i];

        }

      } else if (this.mainObjective == 'profitForMinSharpe') {

        if (populationProfits[i] > maxFitness[0] && populationSharpes[i] >= this.minSharpe) {

          maxFitness = [populationProfits[i], populationSharpes[i], populationScores[i], i];

        }

      }

      fitnessSum += populationProfits[i];

    }

    if (fitnessSum === 0) {

      for (let j = 0; j < this.populationAmt; j++) {

        selectionProb[j] = 1 / this.populationAmt;

      }

    } else {
      for (let j = 0; j < this.populationAmt; j++) {
        selectionProb[j] = populationProfits[j] / fitnessSum;
      }

    }

    let newPopulation = [];

    while (newPopulation.length < this.populationAmt * (1 - this.variation)) {

      let a, b;
      let selectedProb = randomExt.float(1, 0);

      for (let k = 0; k < this.populationAmt; k++) {

        selectedProb -= selectionProb[k];

        if (selectedProb <= 0) {

          a = population[k];
          break;

        }

      }
      selectedProb = randomExt.float(1, 0);

      for (let k = 0; k < this.populationAmt; k++) {

        selectedProb -= selectionProb[k];

        if (selectedProb <= 0) {

          b = population[k];
          break;

        }

      }

      let res = this.crossover(this.mutate(a, this.mutateElements), this.mutate(b, this.mutateElements));
      newPopulation.push(res[0]);
      newPopulation.push(res[1]);

    }

    for (let l = 0; l < this.populationAmt * this.variation; l++) {

      newPopulation.push(this.createGene('all'));

    }

    return [newPopulation, maxFitness];
  }

  getConfig(data) {

    const conf = Object.assign({}, this.baseConfig);

    conf[this.stratName] = Object.keys(data).reduce((acc, key) => {
      acc[key] = data[key];
      return acc;
    }, {});

    Object.assign(conf.tradingAdvisor, {
      candleSize: data.candleSize,
      historySize: data.historySize
    });

    return conf;

  }

  // Calls api for every element in testSeries and returns gain for each
  async fitnessApi(testsSeries) {

    const numberOfParallelQueries = this.parallelqueries;

    const results = await this.queue(testsSeries, numberOfParallelQueries, async (data) => {

      const outconfig = this.getConfig(data);
      const body = await rp.post({
        url: `${this.apiUrl}/api/backtest`,
        json: true,
        body: outconfig,
        headers: { 'Content-Type': 'application/json' },
        timeout: 3600000
      });

      // These properties will be outputted every epoch, remove property if not needed
      const properties = ['balance', 'profit', 'sharpe', 'market', 'relativeProfit', 'yearlyProfit', 'relativeYearlyProfit', 'startPrice', 'endPrice', 'trades'];
      const report = body.performanceReport;
      let result = { profit: 0, metrics: false };

      if (report) {

        let picked = properties.reduce((o, k) => {

          o[k] = report[k];

          return o;

        }, {});

        result = { profit: body.performanceReport.profit, sharpe: body.performanceReport.sharpe, metrics: picked };

      }

      return result;

    });

    let scores = [];
    let profits = [];
    let sharpes = [];
    let otherMetrics = [];

    for (let i in results) {

      if (results.hasOwnProperty(i)) {

        scores.push(results[i]['profit'] * results[i]['sharpe']);
        profits.push(results[i]['profit']);
        sharpes.push(results[i]['sharpe']);
        otherMetrics.push(results[i]['metrics']);

      }

    }

    return { scores, profits, sharpes, otherMetrics };

  }

  async run() {
    // Check for old break point
    const loaded_config = await this.loadBreakPoint();
    let population = this.createPopulation();
    let epochNumber = 0;
    let populationScores;
    let populationProfits;
    let populationSharpes;
    let otherPopulationMetrics;
    let allTimeMaximum = {
      parameters: {},
      score: -5,
      profit: -5,
      sharpe: -5,
      epochNumber: 0,
      otherMetrics: {}
    };

    if (loaded_config) {

      console.log(`Loaded previous config from ${this.configName}-${this.currency}_${this.asset}.json`);
      this.previousBestParams = loaded_config;

      epochNumber = this.previousBestParams.epochNumber;
      populationScores = this.previousBestParams.score;
      populationProfits = this.previousBestParams.profit;
      populationSharpes = this.previousBestParams.sharpe;
      otherPopulationMetrics = this.previousBestParams.otherMetrics;
      allTimeMaximum = {
        parameters: this.previousBestParams.parameters,
        score: this.previousBestParams.score,
        profit: this.previousBestParams.profit,
        sharpe: this.previousBestParams.sharpe,
        epochNumber: this.previousBestParams.epochNumber,
        otherMetrics: this.previousBestParams.otherMetrics
      };

      console.log('Resuming previous run...');

    } else {

      console.log('No previous run data, starting from scratch!');

    }

    console.log(`Starting GA with epoch populations of ${this.populationAmt}, running ${this.parallelqueries} units at a time!`);

    while (1) {

      const startTime = new Date().getTime();
      const res = await this.fitnessApi(population);

      populationScores = res.scores;
      populationProfits = res.profits;
      populationSharpes = res.sharpes;
      otherPopulationMetrics = res.otherMetrics;

      let endTime = new Date().getTime();
      epochNumber++;
      let results = this.runEpoch(population, populationProfits, populationSharpes, populationScores);
      let newPopulation = results[0];
      let maxResult = results[1];
      let score = maxResult[2];
      let profit = maxResult[0];
      let sharpe = maxResult[1];
      let position = maxResult[3];

      this.notifynewhigh = false;
      if (this.mainObjective == 'score') {
        if (score >= allTimeMaximum.score) {
            this.notifynewhigh = true;
            allTimeMaximum.parameters = population[position];
            allTimeMaximum.otherMetrics = otherPopulationMetrics[position];
            allTimeMaximum.score = score;
            allTimeMaximum.profit = profit;
            allTimeMaximum.sharpe = sharpe;
            allTimeMaximum.epochNumber = epochNumber;

        }
      } else if (this.mainObjective == 'profit') {
        if (profit >= allTimeMaximum.profit) {
            this.notifynewhigh = true;
            allTimeMaximum.parameters = population[position];
            allTimeMaximum.otherMetrics = otherPopulationMetrics[position];
            allTimeMaximum.score = score;
            allTimeMaximum.profit = profit;
            allTimeMaximum.sharpe = sharpe;
            allTimeMaximum.epochNumber = epochNumber;

        }
      } else if (this.mainObjective == 'profitForMinSharpe') {
        if (profit >= allTimeMaximum.profit && sharpe >= this.minSharpe) {
            this.notifynewhigh = true;
            allTimeMaximum.parameters = population[position];
            allTimeMaximum.otherMetrics = otherPopulationMetrics[position];
            allTimeMaximum.score = score;
            allTimeMaximum.profit = profit;
            allTimeMaximum.sharpe = sharpe;
            allTimeMaximum.epochNumber = epochNumber;

        }
      }

      console.log(`
    --------------------------------------------------------------
    Epoch number: ${epochNumber}
    Time it took (seconds): ${(endTime - startTime) / 1000}
    Max score: ${score}
    Max profit: ${profit} ${this.currency}
    Max sharpe: ${sharpe}
    Max profit position: ${position}
    Max parameters:
    `,
        util.inspect(population[position], false, null),
        `
    Other metrics:
    `,
        otherPopulationMetrics[position]);

      // Prints out the whole population with its fitness,
      // useful for finding properties that make no sense and debugging
      // for(let element in population){
      //
      //     console.log('Fitness: '+populationProfits[element]+' Properties:');
      //     console.log(population[element]);
      //
      // }

      console.log(`
    --------------------------------------------------------------
    Global Maximums:
    Score: ${allTimeMaximum.score}
    Profit: ${allTimeMaximum.profit} ${this.currency}
    Sharpe: ${allTimeMaximum.sharpe}
    parameters: \n\r`,
    util.inspect(allTimeMaximum.parameters, false, null),
    `
    Global maximum so far:
    `,
        allTimeMaximum.otherMetrics,
        `
    --------------------------------------------------------------
    `);

      // store in json
      const json = JSON.stringify(allTimeMaximum);
      await fs.writeFile(`./results/${this.configName}-${this.currency}_${this.asset}.json`, json, 'utf8').catch(err => console.log(err) );

      if (this.sendemail && this.notifynewhigh) {
        var transporter = nodemailer.createTransport({
          service: this.senderservice,
          auth: {
            user: this.sender,
            pass: this.senderpass
          }
        });
        var mailOptions = {
          from: this.sender,
          to: this.receiver,
          subject: `Profit: ${allTimeMaximum.profit} ${this.currency}`,
          text: json
        };
        transporter.sendMail(mailOptions, function(error, info){
          if (error) {
            console.log(error);
          } else {
            console.log('Email sent: ' + info.response);
          }
        });
      }


      population = newPopulation;

    }

    console.log(`Finished!
  All time maximum:
  ${allTimeMaximum}`);

  }

}


module.exports = Ga;


================================================
FILE: package.json
================================================
{
  "name": "gekkoga",
  "version": "2.0.0",
  "description": "GA for Gekko",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/gekkowarez/gekkoga.git"
  },
  "keywords": [
    "gekkoga"
  ],
  "author": "gekkowarez",
  "license": "MIT",
  "bugs": {
    "url": "https://github.com/gekkowarez/gekkoga/issues"
  },
  "homepage": "https://github.com/gekkowarez/gekkoga#readme",
  "dependencies": {
    "async": "^2.6.0",
    "commander": "^2.11.0",
    "flat": "^4.0.0",
    "fs-extra": "^4.0.2",
    "nodemailer": "^4.4.1",
    "random-ext": "^2.6.1",
    "request": "^2.83.0",
    "request-promise": "^4.2.2"
  }
}


================================================
FILE: results/.gitkeep
================================================


================================================
FILE: run.js
================================================
const Ga = require('./');
const fs = require('fs');

var runGA = require('commander');

runGA
  .version('0.0.3?')
  .option('-c --config <config file>', 'config')
  .parse(process.argv);

if(!runGA.config||!fs.existsSync(runGA.config)){
  console.error("\n"," error: option `-c --config <config file>' argument or file missing","\n");
  process.exit(1);
}

const config = require('./'+runGA.config);
const ga = new Ga(config,runGA.config);

ga.run().catch(err => console.error(err) );