[
  {
    "path": ".gitignore",
    "content": "*~\n*.swp\n\n*.class\n\n# osx\n.DS_Store\n\n# Eclipse build artifacts\n.project\n.classpath\n\n# Maven\ntarget\n\nbuild/\n**/build/\nclasses\n\n# IntelliJ\n.idea/\n*.iml\n*.ipr\n*.iws\n**/*.iml\nout/\n**/.classpath\n**/.project\n.settings/\n**/.settings/\n\n"
  },
  {
    "path": "LICENSE",
    "content": "Copyright (c) 2012-2019, Yin Lou\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n    * Redistributions of source code must retain the above copyright\n      notice, this list of conditions and the following disclaimer.\n    * Redistributions in binary form must reproduce the above copyright\n      notice, this list of conditions and the following disclaimer in the\n      documentation and/or other materials provided with the distribution.\n    * Neither the name of the organization nor the\n      names of its contributors may be used to endorse or promote products\n      derived from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
  },
  {
    "path": "README.md",
    "content": "# Machine Learning Tool Kit\n\nMLTK is a collection of various supervised machine learning algorithms, which is designed for directly training models and further development. For questions or suggestions with the code, please email <a href=\"mailto:machinelearningtoolkit@gmail.com\">machinelearningtoolkit@gmail.com</a>. \n\nSee [wiki](https://github.com/yinlou/mltk/wiki) for full documentation, examples and other information.\n"
  },
  {
    "path": "pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n  xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n  <modelVersion>4.0.0</modelVersion>\n  <groupId>com.github.yinlou</groupId>\n  <artifactId>mltk</artifactId>\n  <version>0.1.1-SNAPSHOT</version>\n\n  <name>mltk</name>\n  <description>Machine Learning Tool Kit</description>\n  <url>https://github.com/yinlou/mltk</url>\n\n  <licenses>\n    <license>\n      <name>BSD 3-Clause</name>\n      <url>https://opensource.org/licenses/BSD-3-Clause</url>\n    </license>\n  </licenses>\n\n   <developers>\n    <developer>\n      <name>Yin Lou</name>\n      <email>machinelearningtoolkit@gmail.com</email>\n    </developer>\n  </developers>\n\n  <scm>\n    <connection>scm:git:git://github.com/yinlou/mltk.git</connection>\n    <developerConnection>scm:git:ssh://github.com:yinlou/mltk.git</developerConnection>\n    <url>https://github.com/yinlou/mltk.git</url>\n  </scm>\n\n  <properties>\n    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>\n    <maven.compiler.source>1.8</maven.compiler.source>\n    <maven.compiler.target>1.8</maven.compiler.target>\n  </properties>\n\n  <profiles>\n    <profile>\n      <id>doclint-java8-disable</id>\n      <activation>\n        <jdk>[1.8,)</jdk>\n      </activation>\n      <properties>\n        <javadoc.opts>-Xdoclint:none</javadoc.opts>\n      </properties>\n    </profile>\n  </profiles>\n\n  <distributionManagement>\n    <snapshotRepository>\n      <id>ossrh</id>\n      <url>https://oss.sonatype.org/content/repositories/snapshots</url>\n    </snapshotRepository>\n  </distributionManagement>\n\n  <build>\n    <plugins>\n      <plugin>\n        <groupId>org.sonatype.plugins</groupId>\n        <artifactId>nexus-staging-maven-plugin</artifactId>\n        <version>1.6.7</version>\n        <extensions>true</extensions>\n        <configuration>\n          <serverId>ossrh</serverId>\n          <nexusUrl>https://oss.sonatype.org/</nexusUrl>\n          <autoReleaseAfterClose>true</autoReleaseAfterClose>\n        </configuration>\n      </plugin>\n      <plugin>\n        <groupId>org.apache.maven.plugins</groupId>\n        <artifactId>maven-compiler-plugin</artifactId>\n        <version>3.5.1</version>\n        <configuration>\n          <showDeprecation>true</showDeprecation>\n          <source>${maven.compiler.source}</source>\n          <target>${maven.compiler.target}</target> \n        </configuration>\n      </plugin>\n      <plugin>\n      <groupId>org.apache.maven.plugins</groupId>\n        <artifactId>maven-source-plugin</artifactId>\n        <version>2.2.1</version>\n        <executions>\n          <execution>\n           <id>attach-sources</id>\n            <goals>\n              <goal>jar-no-fork</goal>\n            </goals>\n          </execution>\n        </executions>\n      </plugin>\n      <plugin>\n        <groupId>org.apache.maven.plugins</groupId>\n        <artifactId>maven-javadoc-plugin</artifactId>\n        <version>2.10.3</version>\n        <configuration>\n          <source>8</source>\n          <doctitle>Machine Learning Tool Kit ${project.version} API</doctitle>\n          <windowtitle>Machine Learning Tool Kit ${project.version} API</windowtitle>\n          <additionalparam>${javadoc.opts}</additionalparam>\n        </configuration>\n        <executions>\n          <execution>\n            <id>attach-javadocs</id>\n            <phase>package</phase>\n            <goals>\n              <goal>jar</goal>\n            </goals>\n          </execution>\n        </executions>\n      </plugin>\n      <plugin>\n      <groupId>org.apache.maven.plugins</groupId>\n        <artifactId>maven-gpg-plugin</artifactId>\n        <version>1.5</version>\n        <executions>\n         <execution>\n            <id>sign-artifacts</id>\n            <phase>verify</phase>\n            <goals>\n             <goal>sign</goal>\n            </goals>\n          </execution>\n        </executions>\n      </plugin>\n    </plugins>\n  </build>\n\n  <dependencies>\n    <dependency>\n      <groupId>junit</groupId>\n      <artifactId>junit</artifactId>\n      <version>4.13.1</version>\n    </dependency>\n  </dependencies>\n\n  <repositories>\n  </repositories>\n\n</project>\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/Argument.java",
    "content": "package mltk.cmdline;\n\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n * Command line argument.\n * \n * @author Yin Lou\n * \n */\n@Retention(java.lang.annotation.RetentionPolicy.RUNTIME)\n@Target(ElementType.FIELD)\npublic @interface Argument {\n\n\t/**\n\t * Name of this argument.\n\t * \n\t * @return the name of this argument.\n\t */\n\tString name() default \"\";\n\n\t/**\n\t * Description of this argument.\n\t * \n\t * @return the description of this argument.\n\t */\n\tString description() default \"\";\n\n\t/**\n\t * Whether this argument is required.\n\t * \n\t * @return {@code true} if the argument is required.\n\t */\n\tboolean required() default false;\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/CmdLineParser.java",
    "content": "package mltk.cmdline;\r\n\r\nimport java.lang.reflect.Field;\r\nimport java.util.ArrayList;\r\nimport java.util.HashMap;\r\nimport java.util.List;\r\nimport java.util.Map;\r\n\r\n/**\r\n * Class for command line parser.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class CmdLineParser {\r\n\r\n\tprivate String name;\r\n\tprivate Object obj;\r\n\r\n\tprivate List<Argument> argList;\r\n\tprivate List<Field> fieldList;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param clazz the class object.\r\n\t * @param obj the object.\r\n\t */\r\n\tpublic CmdLineParser(Class<?> clazz, Object obj) {\r\n\t\tthis.name = clazz.getCanonicalName();\r\n\t\tthis.obj = obj;\r\n\r\n\t\targList = new ArrayList<>();\r\n\t\tfieldList = new ArrayList<>();\r\n\t\t\r\n\t\tprocessFields(obj.getClass().getFields());\r\n\t\tprocessFields(obj.getClass().getDeclaredFields());\r\n\t}\r\n\r\n\t/**\r\n\t * Parses the command line arguments.\r\n\t * \r\n\t * @param args the command line arguments.\r\n\t * @throws IllegalArgumentException\r\n\t * @throws IllegalAccessException\r\n\t */\r\n\tpublic void parse(String[] args) throws IllegalArgumentException, IllegalAccessException {\r\n\t\tif (args.length % 2 != 0) {\r\n\t\t\tthrow new IllegalArgumentException();\r\n\t\t}\r\n\t\tMap<String, String> map = new HashMap<>();\r\n\t\tfor (int i = 0; i < args.length; i += 2) {\r\n\t\t\tmap.put(args[i], args[i + 1]);\r\n\t\t}\r\n\t\tfor (int i = 0; i < argList.size(); i++) {\r\n\t\t\tField field = fieldList.get(i);\r\n\t\t\tArgument arg = argList.get(i);\r\n\t\t\tString value = map.get(arg.name());\r\n\t\t\tif (value != null) {\r\n\t\t\t\tClass<? extends Object> fclass = field.getType();\r\n\t\t\t\tfield.setAccessible(true);\r\n\t\t\t\tif (fclass == String.class) {\r\n\t\t\t\t\tfield.set(obj, value);\r\n\t\t\t\t} else if (fclass == int.class) {\r\n\t\t\t\t\tfield.setInt(obj, Integer.parseInt(value));\r\n\t\t\t\t} else if (fclass == double.class) {\r\n\t\t\t\t\tfield.setDouble(obj, Double.parseDouble(value));\r\n\t\t\t\t} else if (fclass == float.class) {\r\n\t\t\t\t\tfield.setFloat(obj, Float.parseFloat(value));\r\n\t\t\t\t} else if (fclass == boolean.class) {\r\n\t\t\t\t\tfield.setBoolean(obj, Boolean.parseBoolean(value));\r\n\t\t\t\t} else if (fclass == long.class) {\r\n\t\t\t\t\tfield.setLong(obj, Long.parseLong(value));\r\n\t\t\t\t} else if (fclass == char.class) {\r\n\t\t\t\t\tfield.setChar(obj, value.charAt(0));\r\n\t\t\t\t}\r\n\t\t\t} else if (arg.required()) {\r\n\t\t\t\tthrow new IllegalArgumentException();\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Prints the generated usage.\r\n\t */\r\n\tpublic void printUsage() {\r\n\t\tStringBuilder sb = new StringBuilder();\r\n\t\tsb.append(\"Usage: \").append(name).append(\"\\n\");\r\n\r\n\t\tStringBuilder required = new StringBuilder();\r\n\t\tStringBuilder optional = new StringBuilder();\r\n\r\n\t\tfor (Argument arg : argList) {\r\n\t\t\tif (arg.required()) {\r\n\t\t\t\trequired.append(arg.name()).append(\"\\t\").append(arg.description()).append(\"\\n\");\r\n\t\t\t} else {\r\n\t\t\t\toptional.append(\"[\").append(arg.name()).append(\"]\\t\").append(arg.description()).append(\"\\n\");\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tsb.append(required).append(optional);\r\n\t\tSystem.err.println(sb.toString());\r\n\t}\r\n\t\r\n\tprivate void processFields(Field[] fields) {\r\n\t\tfor (Field field : fields) {\r\n\t\t\tArgument argument = field.getAnnotation(Argument.class);\r\n\t\t\tif (argument != null) {\r\n\t\t\t\tfieldList.add(field);\r\n\t\t\t\targList.add(argument);\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/options/HoldoutValidatedLearnerOptions.java",
    "content": "package mltk.cmdline.options;\n\nimport mltk.cmdline.Argument;\n\npublic class HoldoutValidatedLearnerOptions extends LearnerOptions {\n\n\t@Argument(name = \"-v\", description = \"valid set path\")\n\tpublic String validPath = null;\n\t\n\t@Argument(name = \"-e\", description = \"evaluation metric (default: default metric of task)\")\n\tpublic String metric = null;\n\t\n\t@Argument(name = \"-S\", description = \"convergence criteria (default: -1) \")\n\tpublic String cc = \"-1\";\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/options/HoldoutValidatedLearnerWithTaskOptions.java",
    "content": "package mltk.cmdline.options;\n\nimport mltk.cmdline.Argument;\n\npublic class HoldoutValidatedLearnerWithTaskOptions extends HoldoutValidatedLearnerOptions {\n\n\t@Argument(name = \"-g\", description = \"task between classification (c) and regression (r) (default: r)\")\n\tpublic String task = \"r\";\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/options/LearnerOptions.java",
    "content": "package mltk.cmdline.options;\n\nimport mltk.cmdline.Argument;\n\npublic class LearnerOptions {\n\n\t@Argument(name = \"-r\", description = \"attribute file path\")\n\tpublic String attPath = null;\n\n\t@Argument(name = \"-t\", description = \"train set path\", required = true)\n\tpublic String trainPath = null;\n\n\t@Argument(name = \"-o\", description = \"output model path\")\n\tpublic String outputModelPath = null;\n\t\n\t@Argument(name = \"-V\", description = \"verbose (default: true)\")\n\tpublic boolean verbose = true;\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/options/LearnerWithTaskOptions.java",
    "content": "package mltk.cmdline.options;\n\nimport mltk.cmdline.Argument;\n\npublic class LearnerWithTaskOptions extends LearnerOptions {\n\n\t@Argument(name = \"-g\", description = \"task between classification (c) and regression (r) (default: r)\")\n\tpublic String task = \"r\";\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/cmdline/options/package-info.java",
    "content": "/**\n * Provides classes for command line options.\n *\n */\npackage mltk.cmdline.options;"
  },
  {
    "path": "src/main/java/mltk/cmdline/package-info.java",
    "content": "/**\n * Provides classes for command line parser.\n */\npackage mltk.cmdline;"
  },
  {
    "path": "src/main/java/mltk/core/Attribute.java",
    "content": "package mltk.core;\n\n/**\n * Abstract class for attributes.\n * \n * @author Yin Lou\n * \n */\npublic abstract class Attribute implements Comparable<Attribute>, Copyable<Attribute> {\n\n\tpublic enum Type {\n\t\t/**\n\t\t * Nominal type.\n\t\t */\n\t\tNOMINAL,\n\t\t/**\n\t\t * Numeric type.\n\t\t */\n\t\tNUMERIC,\n\t\t/**\n\t\t * Binned type.\n\t\t */\n\t\tBINNED;\n\t}\n\n\tprotected Type type;\n\n\tprotected int index;\n\n\tprotected String name;\n\n\t/**\n\t * Returns the type of this attribute.\n\t * \n\t * @return the type of this attribute.\n\t */\n\tpublic final Type getType() {\n\t\treturn type;\n\t}\n\n\t/**\n\t * Returns the index of this attribute.\n\t * \n\t * @return the index of this attribute.\n\t */\n\tpublic final int getIndex() {\n\t\treturn index;\n\t}\n\n\t/**\n\t * Sets the index of this attribute.\n\t * \n\t * @param index the new index.\n\t */\n\tpublic final void setIndex(int index) {\n\t\tthis.index = index;\n\t}\n\n\t/**\n\t * Returns the name of this attribute.\n\t * \n\t * @return the name of this attribute.\n\t */\n\tpublic final String getName() {\n\t\treturn name;\n\t}\n\n\t@Override\n\tpublic int compareTo(Attribute att) {\n\t\treturn (this.index - att.index);\n\t}\n\n\t@Override\n\tpublic int hashCode() {\n\t\tfinal int prime = 31;\n\t\tint result = 1;\n\t\tresult = prime * result + index;\n\t\tresult = prime * result + ((name == null) ? 0 : name.hashCode());\n\t\tresult = prime * result + ((type == null) ? 0 : type.hashCode());\n\t\treturn result;\n\t}\n\n\t@Override\n\tpublic boolean equals(Object obj) {\n\t\tif (this == obj)\n\t\t\treturn true;\n\t\tif (obj == null)\n\t\t\treturn false;\n\t\tif (getClass() != obj.getClass())\n\t\t\treturn false;\n\t\tAttribute other = (Attribute) obj;\n\t\tif (index != other.index)\n\t\t\treturn false;\n\t\tif (name == null) {\n\t\t\tif (other.name != null)\n\t\t\t\treturn false;\n\t\t} else if (!name.equals(other.name))\n\t\t\treturn false;\n\t\tif (type != other.type)\n\t\t\treturn false;\n\t\treturn true;\n\t}\n\n\t@Override\n\tpublic String toString() {\n\t\treturn name;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/BinnedAttribute.java",
    "content": "package mltk.core;\r\n\r\nimport java.util.Arrays;\r\n\r\nimport mltk.util.ArrayUtils;\r\n\r\n/**\r\n * Class for discretized attributes.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BinnedAttribute extends Attribute {\r\n\r\n\tprotected int numBins;\r\n\tprotected Bins bins;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param name the name of this attribute.\r\n\t * @param numBins number of bins for this attribute.\r\n\t */\r\n\tpublic BinnedAttribute(String name, int numBins) {\r\n\t\tthis(name, numBins, -1);\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param name the name of this attribute.\r\n\t * @param numBins number of bins for this attribute.\r\n\t * @param index the index of this attribute.\r\n\t */\r\n\tpublic BinnedAttribute(String name, int numBins, int index) {\r\n\t\tthis.name = name;\r\n\t\tthis.numBins = numBins;\r\n\t\tthis.bins = null;\r\n\t\tthis.index = index;\r\n\t\tthis.type = Type.BINNED;\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param name the name of this attribute.\r\n\t * @param bins bins for this attribute.\r\n\t */\r\n\tpublic BinnedAttribute(String name, Bins bins) {\r\n\t\tthis(name, bins, -1);\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param name the name of this attribute.\r\n\t * @param bins bins for this attribute.\r\n\t * @param index the index of this attribute.\r\n\t */\r\n\tpublic BinnedAttribute(String name, Bins bins, int index) {\r\n\t\tthis(name, bins.size());\r\n\t\tthis.bins = bins;\r\n\t\tthis.index = index;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BinnedAttribute copy() {\r\n\t\tBinnedAttribute copy = (bins == null ? new BinnedAttribute(name, numBins) : new BinnedAttribute(name, bins));\r\n\t\tcopy.index = index;\r\n\t\treturn copy;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the number of bins.\r\n\t * \r\n\t * @return the number of bins.\r\n\t */\r\n\tpublic int getNumBins() {\r\n\t\treturn numBins;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the bins.\r\n\t * \r\n\t * @return the bins.\r\n\t */\r\n\tpublic Bins getBins() {\r\n\t\treturn bins;\r\n\t}\r\n\r\n\tpublic String toString() {\r\n\t\tif (bins == null) {\r\n\t\t\treturn name + \": binned (\" + numBins + \")\";\r\n\t\t} else {\r\n\t\t\treturn name + \": binned (\" + bins.size() + \";\" + Arrays.toString(bins.boundaries) + \";\"\r\n\t\t\t\t\t+ Arrays.toString(bins.medians) + \")\";\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Parses a binned attribute object from a string.\r\n\t * \r\n\t * @param str the string.\r\n\t * @return a parsed binned attribute.\r\n\t */\r\n\tpublic static BinnedAttribute parse(String str) {\r\n\t\tString[] data = str.split(\": \");\r\n\t\tint start = data[1].indexOf('(') + 1;\r\n\t\tint end = data[1].indexOf(')');\r\n\t\tString[] strs = data[1].substring(start, end).split(\";\");\r\n\t\tint numBins = Integer.parseInt(strs[0]);\r\n\t\tif (strs.length == 1) {\r\n\t\t\treturn new BinnedAttribute(data[0], numBins);\r\n\t\t} else {\r\n\t\t\tdouble[] boundaries = ArrayUtils.parseDoubleArray(strs[1]);\r\n\t\t\tdouble[] medians = ArrayUtils.parseDoubleArray(strs[2]);\r\n\t\t\tBins bins = new Bins(boundaries, medians);\r\n\t\t\treturn new BinnedAttribute(data[0], bins);\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/Bins.java",
    "content": "package mltk.core;\r\n\r\nimport mltk.util.ArrayUtils;\r\n\r\n/**\r\n * Class for bins. Each bin is defined as its upper bound and median.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Bins {\r\n\r\n\t/**\r\n\t * The upper bounds for each bin\r\n\t */\r\n\tprotected double[] boundaries;\r\n\r\n\t/**\r\n\t * The medians for each bin\r\n\t */\r\n\tprotected double[] medians;\r\n\r\n\tprotected Bins() {\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param boundaries the uppber bounds for each bin.\r\n\t * @param medians the medians for each bin.\r\n\t */\r\n\tpublic Bins(double[] boundaries, double[] medians) {\r\n\t\tif (boundaries.length != medians.length) {\r\n\t\t\tthrow new IllegalArgumentException(\"Boundary size doesn't match medians size\");\r\n\t\t}\r\n\t\tthis.boundaries = boundaries;\r\n\t\tthis.medians = medians;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the number of bins.\r\n\t * \r\n\t * @return the number of bins.\r\n\t */\r\n\tpublic int size() {\r\n\t\treturn boundaries.length;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the bin index given a real value using binary search.\r\n\t * \r\n\t * @param value the real value to discretize.\r\n\t * @return the discretized index.\r\n\t */\r\n\tpublic int getIndex(double value) {\r\n\t\tif (value < boundaries[0]) {\r\n\t\t\treturn 0;\r\n\t\t} else if (value >= boundaries[boundaries.length - 1]) {\r\n\t\t\treturn boundaries.length - 1;\r\n\t\t} else {\r\n\t\t\treturn ArrayUtils.findInsertionPoint(boundaries, value);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the median of a bin.\r\n\t * \r\n\t * @param index the index of the bin.\r\n\t * @return the median of the bin.\r\n\t */\r\n\tpublic double getValue(int index) {\r\n\t\treturn medians[index];\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the upper bounds for each bin.\r\n\t * \r\n\t * @return the upper bounds for each bin.\r\n\t */\r\n\tpublic double[] getBoundaries() {\r\n\t\treturn boundaries;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the medians for each bin.\r\n\t * \r\n\t * @return the medians for each bin.\r\n\t */\r\n\tpublic double[] getMedians() {\r\n\t\treturn medians;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/Copyable.java",
    "content": "package mltk.core;\n\n/**\n * Copyable interface.\n * \n * @author Yin Lou\n * \n * @param <T> the type of the object.\n */\npublic interface Copyable<T> {\n\n\t/**\n\t * Returns a (deep) copy of the object.\n\t * \n\t * @return a (deep) copy of the object.\n\t */\n\tpublic T copy();\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/DenseVector.java",
    "content": "package mltk.core;\r\n\r\nimport java.util.Arrays;\r\n\r\n/**\r\n * Class for dense vectors.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class DenseVector implements Vector {\r\n\r\n\tprotected double[] values;\r\n\r\n\t/**\r\n\t * Constructs a dense vector from a double array.\r\n\t * \r\n\t * @param values the double array.\r\n\t */\r\n\tpublic DenseVector(double[] values) {\r\n\t\tthis.values = values;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double getValue(int index) {\r\n\t\treturn values[index];\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] getValues() {\r\n\t\treturn values;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] getValues(int... indices) {\r\n\t\tdouble[] values = new double[indices.length];\r\n\t\tfor (int i = 0; i < values.length; i++) {\r\n\t\t\tvalues[i] = getValue(indices[i]);\r\n\t\t}\r\n\t\treturn values;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void setValue(int index, double value) {\r\n\t\tvalues[index] = value;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void setValue(int[] indices, double[] v) {\r\n\t\tfor (int i = 0; i < indices.length; i++) {\r\n\t\t\tvalues[indices[i]] = v[i];\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic DenseVector copy() {\r\n\t\tdouble[] copyValues = Arrays.copyOf(values, values.length);\r\n\t\treturn new DenseVector(copyValues);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean isSparse() {\r\n\t\treturn false;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/Instance.java",
    "content": "package mltk.core;\n\nimport mltk.util.MathUtils;\n\n/**\n * Class for instances.\n * \n * @author Yin Lou\n * \n */\npublic class Instance implements Copyable<Instance> {\n\n\tprotected Vector vector;\n\tprotected double[] target;\n\tprotected double weight;\n\n\t/**\n\t * Constructs a dense instance from values, target and weight.\n\t * \n\t * @param values the values.\n\t * @param target the target.\n\t * @param weight the weight.\n\t */\n\tpublic Instance(double[] values, double target, double weight) {\n\t\tthis.vector = new DenseVector(values);\n\t\tthis.target = new double[] { target };\n\t\tthis.weight = weight;\n\t}\n\n\t/**\n\t * Constructs a sparse instance from indices, values, target and weight.\n\t * \n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param target the target.\n\t * @param weight the weight.\n\t */\n\tpublic Instance(int[] indices, double[] values, double target, double weight) {\n\t\tthis.vector = new SparseVector(indices, values);\n\t\tthis.target = new double[] { target };\n\t\tthis.weight = weight;\n\t}\n\n\t/**\n\t * Constructs a dense instance from vector, target and weight.\n\t * \n\t * @param vector the vector.\n\t * @param target the target.\n\t * @param weight the weight.\n\t */\n\tpublic Instance(Vector vector, double target, double weight) {\n\t\tthis.vector = vector;\n\t\tthis.target = new double[] { target };\n\t\tthis.weight = weight;\n\t}\n\n\t/**\n\t * Constructor with default weight 1.0.\n\t * \n\t * @param values the values.\n\t * @param target the target.\n\t */\n\tpublic Instance(double[] values, double target) {\n\t\tthis(values, target, 1.0);\n\t}\n\n\t/**\n\t * Constructor with default weight 1.0.\n\t * \n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param target the target.\n\t */\n\tpublic Instance(int[] indices, double[] values, double target) {\n\t\tthis(indices, values, target, 1.0);\n\t}\n\n\t/**\n\t * Construct with default weight 1.0.\n\t * \n\t * @param vector the vector.\n\t * @param target the target.\n\t */\n\tpublic Instance(Vector vector, double target) {\n\t\tthis(vector, target, 1.0);\n\t}\n\n\t/**\n\t * Constructor with default weight 1.0 and no target.\n\t * \n\t * @param values the values.\n\t */\n\tpublic Instance(double[] values) {\n\t\tthis(values, Double.NaN);\n\t}\n\n\t/**\n\t * Constructor with default weight 1.0 and no target.\n\t * \n\t * @param indices the indices.\n\t * @param values the values.\n\t */\n\tpublic Instance(int[] indices, double[] values) {\n\t\tthis(indices, values, Double.NaN);\n\t}\n\n\t/**\n\t * Constructor with default weight 1.0 and no target.\n\t * \n\t * @param vector the vector.\n\t * @param values the values.\n\t */\n\tpublic Instance(Vector vector, double[] values) {\n\t\tthis(vector, Double.NaN);\n\t}\n\n\t/**\n\t * Copy constructor.\n\t * \n\t * @param instance the other instance to copy.\n\t */\n\tpublic Instance(Instance instance) {\n\t\tthis.vector = instance.vector;\n\t\tthis.weight = instance.weight;\n\t\tthis.target = instance.target;\n\t}\n\n\t/**\n\t * Returns {@code true} if the instance is sparse.\n\t * \n\t * @return {@code true} if the instance is sparse.\n\t */\n\tpublic boolean isSparse() {\n\t\treturn vector.isSparse();\n\t}\n\n\t/**\n\t * Returns the value at specified attribute.\n\t * \n\t * @param attIndex the attribute index.\n\t * @return the value at specified attribute.\n\t */\n\tpublic final double getValue(int attIndex) {\n\t\treturn vector.getValue(attIndex);\n\t}\n\n\t/**\n\t * Returns the values.\n\t * \n\t * @return the values.\n\t */\n\tpublic final double[] getValues() {\n\t\treturn vector.getValues();\n\t}\n\n\t/**\n\t * Returns an array representation of values at specified attributes.\n\t * \n\t * @param attributes the attributes.\n\t * @return an array representation of values at specified attributes.\n\t */\n\tpublic final double[] getValues(int... attributes) {\n\t\treturn vector.getValues(attributes);\n\t}\n\n\t/**\n\t * Sets the value at specified attribute.\n\t * \n\t * @param attIndex the attribute index.\n\t * @param value the new value to set.\n\t */\n\tpublic final void setValue(int attIndex, double value) {\n\t\tvector.setValue(attIndex, value);\n\t}\n\n\t/**\n\t * Sets the value at specified attribute.\n\t * \n\t * @param attribute the attribute.\n\t * @param value the new value to set.\n\t */\n\tpublic final void setValue(Attribute attribute, double value) {\n\t\tsetValue(attribute.getIndex(), value);\n\t}\n\n\t/**\n\t * Sets the values at specified attributes.\n\t * \n\t * @param attributes the attribute index array.\n\t * @param v the value array.\n\t */\n\tpublic final void setValue(int[] attributes, double[] v) {\n\t\tfor (int i = 0; i < attributes.length; i++) {\n\t\t\tsetValue(attributes[i], v[i]);\n\t\t}\n\t}\n\n\t@Override\n\tpublic Instance copy() {\n\t\tVector copyVector = vector.copy();\n\t\treturn new Instance(copyVector, target[0], weight);\n\t}\n\n\t/**\n\t * Returns a shallow copy.\n\t * \n\t * @return a shallow copy.\n\t */\n\tpublic Instance clone() {\n\t\treturn new Instance(this);\n\t}\n\n\t/**\n\t * Returns {@code true} if a specific attribute value is missing.\n\t * \n\t * @param attIndex the attribute index.\n\t * @return {@code true} if a specific attribute value is missing.\n\t */\n\tpublic boolean isMissing(int attIndex) {\n\t\treturn Double.isNaN(getValue(attIndex));\n\t}\n\n\t/**\n\t * Returns the value at specified attribute.\n\t * \n\t * @param att the attribute object.\n\t * @return the value at specified attribute.\n\t */\n\tpublic double getValue(Attribute att) {\n\t\treturn getValue(att.getIndex());\n\t}\n\n\t/**\n\t * Returns the vector.\n\t * \n\t * @return the vector.\n\t */\n\tpublic Vector getVector() {\n\t\treturn vector;\n\t}\n\n\t/**\n\t * Returns the weight of this instance.\n\t * \n\t * @return the weight of this instance.\n\t */\n\tpublic double getWeight() {\n\t\treturn weight;\n\t}\n\n\t/**\n\t * Sets the weight of this instance.\n\t * \n\t * @param weight the new weight of this instance.\n\t */\n\tpublic void setWeight(double weight) {\n\t\tthis.weight = weight;\n\t}\n\n\t/**\n\t * Returns the target value.\n\t * \n\t * @return the target value.\n\t */\n\tpublic double getTarget() {\n\t\treturn target[0];\n\t}\n\n\t/**\n\t * Sets the target value.\n\t * \n\t * @param target the new target value.\n\t */\n\tpublic void setTarget(double target) {\n\t\tthis.target[0] = target;\n\t}\n\n\t/**\n\t * Returns the string representation of this instance.\n\t */\n\tpublic String toString() {\n\t\tStringBuilder sb = new StringBuilder();\n\t\tif (isSparse()) {\n\t\t\tsb.append(getTarget());\n\t\t\tSparseVector sv = (SparseVector) vector;\n\t\t\tint[] indices = sv.getIndices();\n\t\t\tdouble[] values = sv.getValues();\n\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\tsb.append(\" \").append(indices[i]).append(\":\");\n\t\t\t\tprint(sb, values[i]);\n\t\t\t}\n\t\t} else {\n\t\t\tdouble[] values = getValues();\n\t\t\tprint(sb, values[0]);\n\t\t\tfor (int i = 1; i < values.length; i++) {\n\t\t\t\tsb.append(\"\\t\");\n\t\t\t\tprint(sb, values[i]);\n\t\t\t}\n\t\t\tif (!Double.isNaN(getTarget())) {\n\t\t\t\tsb.append(\"\\t\");\n\t\t\t\tprint(sb, getTarget());\n\t\t\t}\n\t\t}\n\t\treturn sb.toString();\n\t}\n\t\n\tprotected void print(StringBuilder sb, double v) {\n\t\tif (MathUtils.isInteger(v)) {\n\t\t\tsb.append((int) v);\n\t\t} else {\n\t\t\tsb.append(v);\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/Instances.java",
    "content": "package mltk.core;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\n\nimport mltk.util.Random;\n\n/**\n * Class for handling an ordered set of instances.\n * \n * @author Yin Lou\n * \n */\npublic class Instances implements Iterable<Instance>, Copyable<Instances> {\n\n\tprotected List<Attribute> attributes;\n\tprotected List<Instance> instances;\n\tprotected Attribute targetAtt;\n\n\t/**\n\t * Constructs a dataset from attributes.\n\t * \n\t * @param attributes the attributes.\n\t */\n\tpublic Instances(List<Attribute> attributes) {\n\t\tthis(attributes, null);\n\t}\n\n\t/**\n\t * Constructs a dataset from attributes, with specified capacity.\n\t * \n\t * @param attributes the attributes.\n\t * @param capacity the capacity.\n\t */\n\tpublic Instances(List<Attribute> attributes, int capacity) {\n\t\tthis(attributes, null, capacity);\n\t}\n\n\t/**\n\t * Constructs a dataset from attributes and target attribute.\n\t * \n\t * @param attributes the attributes.\n\t * @param targetAtt the target attribute.\n\t */\n\tpublic Instances(List<Attribute> attributes, Attribute targetAtt) {\n\t\tthis(attributes, targetAtt, 1000);\n\t}\n\n\t/**\n\t * Constructs a dataset from attributes and target attribute, with specified capacity.\n\t * \n\t * @param attributes the attributes.\n\t * @param targetAtt the target attribute.\n\t * @param capacity the capacity.\n\t */\n\tpublic Instances(List<Attribute> attributes, Attribute targetAtt, int capacity) {\n\t\tthis.attributes = attributes;\n\t\tthis.targetAtt = targetAtt;\n\t\tthis.instances = new ArrayList<>(capacity);\n\t}\n\n\t/**\n\t * Copy constructor.\n\t * \n\t * @param instances the instances to copy.\n\t */\n\tpublic Instances(Instances instances) {\n\t\tthis.attributes = instances.attributes;\n\t\tthis.targetAtt = instances.targetAtt;\n\t\tthis.instances = new ArrayList<>(instances.instances);\n\t}\n\n\t/**\n\t * Adds an instance to the end of the dataset.\n\t * \n\t * @param instance the instance to add.\n\t */\n\tpublic void add(Instance instance) {\n\t\tinstances.add(instance);\n\t}\n\n\t/**\n\t * Returns the instance at given index.\n\t * \n\t * @param index the index.\n\t * @return the instance at given index.\n\t */\n\tpublic Instance get(int index) {\n\t\treturn instances.get(index);\n\t}\n\n\t/**\n\t * Returns the target attribute.\n\t * \n\t * @return the target attribute.\n\t */\n\tpublic final Attribute getTargetAttribute() {\n\t\treturn targetAtt;\n\t}\n\n\t/**\n\t * Sets the target attribute.\n\t * \n\t * @param targetAtt the target attribute.\n\t */\n\tpublic final void setTargetAttribute(Attribute targetAtt) {\n\t\tthis.targetAtt = targetAtt;\n\t}\n\n\t@Override\n\tpublic Iterator<Instance> iterator() {\n\t\treturn instances.iterator();\n\t}\n\n\t/**\n\t * Returns the size of this dataset, i.e., the number of instances.\n\t * \n\t * @return the size of this dataset.\n\t */\n\tpublic final int size() {\n\t\treturn instances.size();\n\t}\n\n\t/**\n\t * Returns the dimension of this dataset, i.e., the number of attributes. Note that class attribute does not count.\n\t * \n\t * @return the dimension of this dataset.\n\t */\n\tpublic final int dimension() {\n\t\treturn attributes.size();\n\t}\n\n\t/**\n\t * Returns the list of attributes.\n\t * \n\t * @return the list of attributes.\n\t */\n\tpublic List<Attribute> getAttributes() {\n\t\treturn attributes;\n\t}\n\n\t/**\n\t * Returns the list of attributes at given locations.\n\t * \n\t * @param indices the indices.\n\t * @return the list of attributes at given locations.\n\t */\n\tpublic List<Attribute> getAttributes(int... indices) {\n\t\tList<Attribute> attributes = new ArrayList<>(indices.length);\n\t\tfor (int index : indices) {\n\t\t\tattributes.add(this.attributes.get(index));\n\t\t}\n\t\treturn attributes;\n\t}\n\n\t/**\n\t * Sets the attributes.\n\t * \n\t * @param attributes the attributes to set.\n\t */\n\tpublic void setAttributes(List<Attribute> attributes) {\n\t\tthis.attributes = attributes;\n\t}\n\n\t/**\n\t * Resets this dataset.\n\t */\n\tpublic void clear() {\n\t\tinstances.clear();\n\t}\n\n\t/**\n\t * Randomly permutes this dataset.\n\t */\n\tpublic void shuffle() {\n\t\tCollections.shuffle(instances, Random.getInstance().getRandom());\n\t}\n\n\t/**\n\t * Randomly permutes this dataset.\n\t * \n\t * @param rand the source of randomness to use to shuffle the dataset.\n\t */\n\tpublic void shuffle(java.util.Random rand) {\n\t\tCollections.shuffle(instances, rand);\n\t}\n\n\t@Override\n\tpublic Instances copy() {\n\t\tList<Attribute> attributes = new ArrayList<>(this.attributes);\n\t\tInstances copy = new Instances(attributes, targetAtt, instances.size());\n\t\tfor (Instance instance : instances) {\n\t\t\tcopy.add(instance.copy());\n\t\t}\n\t\treturn copy;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/NominalAttribute.java",
    "content": "package mltk.core;\n\n/**\n * Class for nominal attributes.\n * \n * @author Yin Lou\n * \n */\npublic class NominalAttribute extends Attribute {\n\n\tprotected String[] states;\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param name the name of this attribute.\n\t * @param states the states for this attribute.\n\t */\n\tpublic NominalAttribute(String name, String[] states) {\n\t\tthis(name, states, -1);\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param name the name of this attribute.\n\t * @param states the states for this attribute.\n\t * @param index the index of this attribute.\n\t */\n\tpublic NominalAttribute(String name, String[] states, int index) {\n\t\tthis.name = name;\n\t\tthis.states = states;\n\t\tthis.index = index;\n\t\tthis.type = Type.NOMINAL;\n\t}\n\n\tpublic NominalAttribute copy() {\n\t\tNominalAttribute copy = new NominalAttribute(name, states);\n\t\tcopy.index = this.index;\n\t\treturn copy;\n\t}\n\n\t/**\n\t * Returns the cardinality of this attribute.\n\t * \n\t * @return the cardinality of this attribute.\n\t */\n\tpublic int getCardinality() {\n\t\treturn states.length;\n\t}\n\n\t/**\n\t * Returns the state given an index.\n\t * \n\t * @param index the index.\n\t * @return the state given an index.\n\t */\n\tpublic String getState(int index) {\n\t\treturn states[index];\n\t}\n\n\t/**\n\t * Returns the states.\n\t * \n\t * @return the states.\n\t */\n\tpublic String[] getStates() {\n\t\treturn states;\n\t}\n\n\tpublic String toString() {\n\t\tStringBuilder sb = new StringBuilder();\n\t\tsb.append(name).append(\": {\").append(states[0]);\n\t\tfor (int i = 1; i < states.length; i++) {\n\t\t\tsb.append(\", \").append(states[i]);\n\t\t}\n\t\tsb.append(\"}\");\n\t\treturn sb.toString();\n\t}\n\n\t/**\n\t * Parses a nominal attribute ojbect from a string.\n\t * \n\t * @param str the string.\n\t * @return a parsed nominal attribute.\n\t */\n\tpublic static NominalAttribute parse(String str) {\n\t\tString[] data = str.split(\": \");\n\t\tint start = data[1].indexOf('{') + 1;\n\t\tint end = data[1].indexOf('}');\n\t\tString[] states = data[1].substring(start, end).split(\",\");\n\t\tfor (int j = 0; j < states.length; j++) {\n\t\t\tstates[j] = states[j].trim();\n\t\t}\n\t\treturn new NominalAttribute(data[0], states);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/NumericalAttribute.java",
    "content": "package mltk.core;\n\n/**\n * Class for numerical attributes.\n * \n * @author Yin Lou\n * \n */\npublic class NumericalAttribute extends Attribute {\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param name the name of this attribute.\n\t */\n\tpublic NumericalAttribute(String name) {\n\t\tthis(name, -1);\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param name the name of this attribute.\n\t * @param index the index of this attribute.\n\t */\n\tpublic NumericalAttribute(String name, int index) {\n\t\tthis.name = name;\n\t\tthis.index = index;\n\t\tthis.type = Type.NUMERIC;\n\t}\n\n\tpublic NumericalAttribute copy() {\n\t\tNumericalAttribute copy = new NumericalAttribute(this.name);\n\t\tcopy.index = this.index;\n\t\treturn copy;\n\t}\n\n\tpublic String toString() {\n\t\treturn name + \": cont\";\n\t}\n\n\t/**\n\t * Parses a numerical attribute object from a string.\n\t * \n\t * @param str the string.\n\t * @return a parsed numerical attribute.\n\t */\n\tpublic static NumericalAttribute parse(String str) {\n\t\tString[] data = str.split(\": \");\n\t\treturn new NumericalAttribute(data[0]);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/Sampling.java",
    "content": "package mltk.core;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.util.Permutation;\nimport mltk.util.Random;\nimport mltk.util.tuple.IntPair;\n\n/**\n * Class for creating samples.\n * \n * @author Yin Lou\n * \n */\npublic class Sampling {\n\n\t/**\n\t * Returns a bootstrap sample.\n\t * \n\t * @param instances the data set.\n\t * @return a bootstrap sample.\n\t */\n\tpublic static Instances createBootstrapSample(Instances instances) {\n\t\tRandom rand = Random.getInstance();\n\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tint idx = rand.nextInt(instances.size());\n\t\t\tmap.put(idx, map.getOrDefault(idx, 0) + 1);\n\t\t}\n\t\tInstances bag = new Instances(instances.getAttributes(), instances.getTargetAttribute(), map.size());\n\t\tfor (Integer idx : map.keySet()) {\n\t\t\tint weight = map.get(idx);\n\t\t\tInstance instance = instances.get(idx).clone();\n\t\t\tinstance.setWeight(weight);\n\t\t\tbag.add(instance);\n\t\t}\n\t\treturn bag;\n\t}\n\n\t/**\n\t * Returns a bootstrap sample with out-of-bag samples.\n\t * \n\t * @param instances the data set.\n\t * @param bagIndices the index of sampled instances with weights.\n\t * @param oobIndices the out-of-bag indexes.\n\t */\n\tpublic static void createBootstrapSample(Instances instances, Map<Integer, Integer> bagIndices,\n\t\t\tList<Integer> oobIndices) {\n\t\tRandom rand = Random.getInstance();\n\t\tfor (;;) {\n\t\t\tbagIndices.clear();\n\t\t\toobIndices.clear();\n\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\tint idx = rand.nextInt(instances.size());\n\t\t\t\tbagIndices.put(idx, bagIndices.getOrDefault(idx, 0) + 1);\n\t\t\t}\n\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\tif (!bagIndices.containsKey(i)) {\n\t\t\t\t\toobIndices.add(i);\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (oobIndices.size() > 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns a bootstrap sample of indices and weights.\n\t * \n\t * @param n the size of the dataset to sample.\n\t * @return a bootstrap sample of indices and weights.\n\t */\n\tpublic static IntPair[] createBootstrapSampleIndices(int n) {\n\t\tRandom rand = Random.getInstance();\n\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\tfor (int i = 0; i < n; i++) {\n\t\t\tint idx = rand.nextInt(n);\n\t\t\tmap.put(idx, map.getOrDefault(idx, 0) + 1);\n\t\t}\n\t\tIntPair[] indices = new IntPair[map.size()];\n\t\tint k = 0;\n\t\tfor (Map.Entry<Integer, Integer> entry : map.entrySet()) {\n\t\t\tindices[k++] = new IntPair(entry.getKey(), entry.getValue());\n\t\t}\n\t\treturn indices;\n\t}\n\n\t/**\n\t * Returns a subsample.\n\t * \n\t * @param instances the dataset.\n\t * @param n the sample size.\n\t * @return a subsample.\n\t */\n\tpublic static Instances createSubsample(Instances instances, int n) {\n\t\tPermutation perm = new Permutation(instances.size());\n\t\tperm.permute();\n\t\tint[] a = perm.getPermutation();\n\t\tInstances sample = new Instances(instances.getAttributes(), instances.getTargetAttribute(), n);\n\t\tfor (int i = 0; i < n; i++) {\n\t\t\tsample.add(instances.get(a[i]));\n\t\t}\n\t\treturn sample;\n\t}\n\n\t/**\n\t * Returns a set of bags.\n\t * \n\t * @param instances the dataset.\n\t * @param b the number of bagging iterations.\n\t * @return a set of bags.\n\t */\n\tpublic static Instances[] createBags(Instances instances, int b) {\n\t\tInstances[] bags = null;\n\t\tif (b <= 0) {\n\t\t\t// No bagging\n\t\t\tbags = new Instances[] { instances };\n\t\t} else {\n\t\t\tbags = new Instances[b];\n\t\t\tfor (int i = 0; i < b; i++) {\n\t\t\t\tbags[i] = Sampling.createBootstrapSample(instances);\n\t\t\t}\n\t\t}\n\t\treturn bags;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/SparseVector.java",
    "content": "package mltk.core;\r\n\r\nimport java.util.Arrays;\r\n\r\n/**\r\n * Class for sparse vectors.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class SparseVector implements Vector {\r\n\r\n\tprotected int[] indices;\r\n\tprotected double[] values;\r\n\r\n\t/**\r\n\t * Constructs a sparse vector from sparse-format arrays.\r\n\t * \r\n\t * @param indices the indices array.\r\n\t * @param values the values array.\r\n\t */\r\n\tpublic SparseVector(int[] indices, double[] values) {\r\n\t\tthis.indices = indices;\r\n\t\tthis.values = values;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic SparseVector copy() {\r\n\t\tint[] copyIndices = Arrays.copyOf(indices, indices.length);\r\n\t\tdouble[] copyValues = Arrays.copyOf(values, values.length);\r\n\t\treturn new SparseVector(copyIndices, copyValues);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double getValue(int index) {\r\n\t\tint idx = Arrays.binarySearch(indices, index);\r\n\t\tif (idx >= 0) {\r\n\t\t\treturn values[idx];\r\n\t\t} else {\r\n\t\t\treturn 0;\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] getValues() {\r\n\t\treturn values;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the internal representation of indices.\r\n\t * \r\n\t * @return the internal representation of indices.\r\n\t */\r\n\tpublic int[] getIndices() {\r\n\t\treturn indices;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] getValues(int... indices) {\r\n\t\tdouble[] values = new double[indices.length];\r\n\t\tfor (int i = 0; i < values.length; i++) {\r\n\t\t\tvalues[i] = getValue(indices[i]);\r\n\t\t}\r\n\t\treturn values;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void setValue(int index, double value) {\r\n\t\tint idx = Arrays.binarySearch(indices, index);\r\n\t\tif (idx >= 0) {\r\n\t\t\tvalues[idx] = value;\r\n\t\t} else {\r\n\t\t\tthrow new UnsupportedOperationException();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void setValue(int[] indices, double[] v) {\r\n\t\tfor (int i = 0; i < indices.length; i++) {\r\n\t\t\tsetValue(indices[i], v[i]);\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean isSparse() {\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/Vector.java",
    "content": "package mltk.core;\n\n/**\n * Interface for vectors.\n * \n * @author Yin Lou\n * \n */\npublic interface Vector extends Copyable<Vector> {\n\n\t/**\n\t * Returns the value at specified index.\n\t * \n\t * @param index the index.\n\t * @return the value at specified index.\n\t */\n\tpublic double getValue(int index);\n\n\t/**\n\t * Returns the internal representation of values.\n\t * \n\t * @return the internal representation of values.\n\t */\n\tpublic double[] getValues();\n\n\t/**\n\t * Returns an array representation of values at specified indices.\n\t * \n\t * @param indices the indices.\n\t * @return an array representation of values at specified indices.\n\t */\n\tpublic double[] getValues(int... indices);\n\n\t/**\n\t * Sets the value at specified index.\n\t * \n\t * @param index the index.\n\t * @param value the new value to set.\n\t */\n\tpublic void setValue(int index, double value);\n\n\t/**\n\t * Sets the values at specified indices.\n\t * \n\t * @param indices the index array.\n\t * @param v the value array.\n\t */\n\tpublic void setValue(int[] indices, double[] v);\n\n\t/**\n\t * Returns {@code true} if the vector is sparse.\n\t * \n\t * @return {@code true} if the vector is sparse.\n\t */\n\tpublic boolean isSparse();\n\n\t/**\n\t * Returns a (deep) copy of the vector.\n\t * \n\t * @return a (deep) copy of the vector.\n\t */\n\tpublic Vector copy();\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/Writable.java",
    "content": "package mltk.core;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\n/**\n * Writable interface.\n * \n * @author Yin Lou\n * \n */\npublic interface Writable {\n\n\t/**\n\t * Reads in this object.\n\t * \n\t * @param in the reader.\n\t * @throws Exception\n\t */\n\tvoid read(BufferedReader in) throws Exception;\n\n\t/**\n\t * Writes this object.\n\t * \n\t * @param out the writer.\n\t * @throws Exception\n\t */\n\tvoid write(PrintWriter out) throws Exception;\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/io/AttributesReader.java",
    "content": "package mltk.core.io;\n\nimport java.io.BufferedReader;\nimport java.io.FileReader;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.NominalAttribute;\nimport mltk.core.NumericalAttribute;\nimport mltk.util.tuple.Pair;\n\n/**\n * Class for reading attributes. It only reads in a list of attributes from the attribute file.\n * \n * @author Yin Lou\n * \n */\npublic class AttributesReader {\n\n\t/**\n\t * Reads attributes and class attribute from attribute file.\n\t * \n\t * @param attFile the attribute file.\n\t * @return a pair of attributes and target attribute (null if no target attribute).\n\t * @throws IOException\n\t */\n\tpublic static Pair<List<Attribute>, Attribute> read(String attFile) throws IOException {\n\t\tBufferedReader br = new BufferedReader(new FileReader(attFile), 65535);\n\t\tPair<List<Attribute>, Attribute> pair = read(br);\n\t\tbr.close();\n\n\t\treturn pair;\n\t}\n\t\n\t/**\n\t * Reads attributes and class attribute from attribute file.\n\t * \n\t * @param br the reader.\n\t * @return a pair of attributes and target attribute (null if no target attribute).\n\t * @throws IOException\n\t */\n\tpublic static Pair<List<Attribute>, Attribute> read(BufferedReader br) throws IOException {\n\t\tList<Attribute> attributes = new ArrayList<Attribute>();\n\t\tAttribute targetAtt = null;\n\t\tSet<String> usedNames = new HashSet<>();\n\t\tfor (int i = 0;; i++) {\n\t\t\tString line = br.readLine();\n\t\t\tif (line == null) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tAttribute att = null;\n\t\t\tif (line.indexOf(\"binned\") != -1) {\n\t\t\t\tatt = BinnedAttribute.parse(line);\n\t\t\t} else if (line.indexOf(\"{\") != -1) {\n\t\t\t\tatt = NominalAttribute.parse(line);\n\t\t\t} else {\n\t\t\t\tatt = NumericalAttribute.parse(line);\n\t\t\t}\n\t\t\tatt.setIndex(i);\n\t\t\tif (line.indexOf(\" (target)\") != -1) {\n\t\t\t\ttargetAtt = att;\n\t\t\t\ti--;\n\t\t\t} else {\n\t\t\t\tif (usedNames.contains(att.getName())) {\n\t\t\t\t\tthrow new RuntimeException(\"Duplicate attribute name: \" + att.getName());\n\t\t\t\t}\n\t\t\t\tusedNames.add(att.getName());\n\t\t\t\tattributes.add(att);\n\t\t\t}\n\t\t\tif (line.indexOf(\" (x)\") != -1) {\n\t\t\t\tatt.setIndex(-1);\n\t\t\t}\n\t\t}\n\n\t\treturn new Pair<List<Attribute>, Attribute>(attributes, targetAtt);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/io/InstancesReader.java",
    "content": "package mltk.core.io;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.FileReader;\r\nimport java.io.IOException;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\nimport java.util.TreeSet;\r\n\r\nimport mltk.core.Attribute;\r\nimport mltk.core.Instance;\r\nimport mltk.core.Instances;\r\nimport mltk.core.NominalAttribute;\r\nimport mltk.core.NumericalAttribute;\r\nimport mltk.util.MathUtils;\r\nimport mltk.util.tuple.Pair;\r\n\r\n/**\r\n * Class for reading instances.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class InstancesReader {\r\n\r\n\t/**\r\n\t * Reads a set of instances from attribute file and data file. Attribute file can be null. Default delimiter is\r\n\t * whitespace.\r\n\t * \r\n\t * @param attFile the attribute file.\r\n\t * @param dataFile the data file.\r\n\t * @return a set of instances.\r\n\t * @throws IOException\r\n\t */\r\n\tpublic static Instances read(String attFile, String dataFile) throws IOException {\r\n\t\treturn read(attFile, dataFile, \"\\\\s+\");\r\n\t}\r\n\r\n\t/**\r\n\t * Reads a set of instances from attribute file and data file. Attribute file can be null.\r\n\t * \r\n\t * @param attFile the attribute file.\r\n\t * @param dataFile the data file.\r\n\t * @param delimiter the delimiter.\r\n\t * @return a set of instances.\r\n\t * @throws IOException\r\n\t */\r\n\tpublic static Instances read(String attFile, String dataFile, String delimiter) throws IOException {\r\n\t\tif (attFile != null) {\r\n\t\t\tPair<List<Attribute>, Attribute> pair = AttributesReader.read(attFile);\r\n\t\t\tint classIndex = -1;\r\n\t\t\tif (pair.v2 != null) {\r\n\t\t\t\tclassIndex = pair.v2.getIndex();\r\n\t\t\t\tpair.v2.setIndex(-1);\r\n\t\t\t}\r\n\t\t\tList<Attribute> attributes = pair.v1;\r\n\t\t\tInstances instances = new Instances(attributes, pair.v2);\r\n\t\t\tint totalLength = instances.dimension();\r\n\t\t\tif (classIndex != -1) {\r\n\t\t\t\ttotalLength++;\r\n\t\t\t}\r\n\t\t\tBufferedReader br = new BufferedReader(new FileReader(dataFile), 65535);\r\n\t\t\tfor (;;) {\r\n\t\t\t\tString line = br.readLine();\r\n\t\t\t\tif (line == null) {\r\n\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t\tString[] data = line.split(delimiter);\r\n\t\t\t\tInstance instance = null;\r\n\t\t\t\tif (data.length >= 2 && data[1].indexOf(':') >= 0) {\r\n\t\t\t\t\t// Sparse instance\r\n\t\t\t\t\tinstance = parseSparseInstance(data);\r\n\t\t\t\t} else if (data.length == totalLength) {\r\n\t\t\t\t\t// Dense instance\r\n\t\t\t\t\tinstance = parseDenseInstance(data, classIndex);\r\n\t\t\t\t} else {\r\n\t\t\t\t\tSystem.err.println(\"Processed as dense vector but the number of attributes provided in the attribute file does not match with the number of columns in this row\");\r\n\t\t\t\t}\r\n\t\t\t\tif (instance != null) {\r\n\t\t\t\t\tinstances.add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tbr.close();\r\n\t\t\t\r\n\t\t\t// Process skipped features\r\n\t\t\tfor (int i = attributes.size() - 1; i >= 0; i--) {\r\n\t\t\t\tif (attributes.get(i).getIndex() < 0) {\r\n\t\t\t\t\tattributes.remove(i);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\t\r\n\t\t\treturn instances;\r\n\t\t} else {\r\n\t\t\tList<Attribute> attributes = new ArrayList<>();\r\n\t\t\tInstances instances = new Instances(attributes);\r\n\t\t\tint totalLength = -1;\r\n\r\n\t\t\tTreeSet<Integer> attrSet = new TreeSet<>();\r\n\t\t\tBufferedReader br = new BufferedReader(new FileReader(dataFile), 65535);\r\n\t\t\tfor (;;) {\r\n\t\t\t\tString line = br.readLine();\r\n\t\t\t\tif (line == null) {\r\n\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t\tString[] data = line.split(delimiter);\r\n\t\t\t\tInstance instance = null;\r\n\t\t\t\tif (data.length >= 2 && data[1].indexOf(':') >= 0) {\r\n\t\t\t\t\t// Sparse instance\r\n\t\t\t\t\tinstance = parseSparseInstance(data, attrSet);\r\n\t\t\t\t} else {\r\n\t\t\t\t\t// Dense instance\r\n\t\t\t\t\tif (totalLength == -1) {\r\n\t\t\t\t\t\ttotalLength = data.length;\r\n\t\t\t\t\t} else if (data.length == totalLength) {\r\n\t\t\t\t\t\tinstance = parseDenseInstance(data, -1);\r\n\t\t\t\t\t}\r\n\r\n\t\t\t\t}\r\n\t\t\t\tif (instance != null) {\r\n\t\t\t\t\tinstances.add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tbr.close();\r\n\r\n\t\t\tif (totalLength == -1) {\r\n\t\t\t\tfor (Integer attIndex : attrSet) {\r\n\t\t\t\t\tAttribute att = new NumericalAttribute(\"f\" + attIndex);\r\n\t\t\t\t\tatt.setIndex(attIndex);\r\n\t\t\t\t\tattributes.add(att);\r\n\t\t\t\t}\r\n\t\t\t} else {\r\n\t\t\t\tfor (int j = 0; j < totalLength; j++) {\r\n\t\t\t\t\tAttribute att = new NumericalAttribute(\"f\" + j);\r\n\t\t\t\t\tatt.setIndex(j);\r\n\t\t\t\t\tattributes.add(att);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\r\n\t\t\tassignTargetAttribute(instances);\r\n\t\t\treturn instances;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Reads a set of dense instances from data file. Default delimiter is whitespace.\r\n\t * \r\n\t * @param file the data file.\r\n\t * @param targetIndex the index of the target attribute, -1 if no target attribute.\r\n\t * @return a set of dense instances.\r\n\t * @throws IOException\r\n\t */\r\n\tpublic static Instances read(String file, int targetIndex) throws IOException {\r\n\t\treturn read(file, targetIndex, \"\\\\s+\");\r\n\t}\r\n\r\n\t/**\r\n\t * Reads a set of dense instances from data file.\r\n\t * \r\n\t * @param file the data file.\r\n\t * @param targetIndex the index of the target attribute, -1 if no target attribute.\r\n\t * @param delimiter the delimiter.\r\n\t * @return a set of dense instances.\r\n\t * @throws IOException\r\n\t */\r\n\tpublic static Instances read(String file, int targetIndex, String delimiter) throws IOException {\r\n\t\tBufferedReader br = new BufferedReader(new FileReader(file), 65535);\r\n\t\tList<Attribute> attributes = new ArrayList<>();\r\n\t\tInstances instances = new Instances(attributes);\r\n\t\tfor (;;) {\r\n\t\t\tString line = br.readLine();\r\n\t\t\tif (line == null) {\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t\tString[] data = line.split(delimiter);\r\n\t\t\tInstance instance = parseDenseInstance(data, targetIndex);\r\n\t\t\tinstances.add(instance);\r\n\t\t}\r\n\t\tbr.close();\r\n\r\n\t\tint numAttributes = instances.get(0).getValues().length;\r\n\t\tfor (int i = 0; i < numAttributes; i++) {\r\n\t\t\tAttribute att = new NumericalAttribute(\"f\" + i);\r\n\t\t\tatt.setIndex(i);\r\n\t\t\tattributes.add(att);\r\n\t\t}\r\n\r\n\t\tif (targetIndex >= 0) {\r\n\t\t\tassignTargetAttribute(instances);\r\n\t\t}\r\n\r\n\t\treturn instances;\r\n\t}\r\n\r\n\t/**\r\n\t * Parses a dense instance from strings.\r\n\t * \r\n\t * @param data the string array.\r\n\t * @param classIndex the class index.\r\n\t * @return a dense instance from strings.\r\n\t */\r\n\tstatic Instance parseDenseInstance(String[] data, int classIndex) {\r\n\t\tdouble classValue = Double.NaN;\r\n\t\tif (classIndex < 0) {\r\n\t\t\tdouble[] vector = new double[data.length];\r\n\t\t\tfor (int i = 0; i < data.length; i++) {\r\n\t\t\t\tvector[i] = parseDouble(data[i]);\r\n\t\t\t}\r\n\t\t\treturn new Instance(vector, classValue);\r\n\t\t} else {\r\n\t\t\tdouble[] vector = new double[data.length - 1];\r\n\t\t\tfor (int i = 0; i < data.length; i++) {\r\n\t\t\t\tdouble value = parseDouble(data[i]);\r\n\t\t\t\tif (i < classIndex) {\r\n\t\t\t\t\tvector[i] = value;\r\n\t\t\t\t} else if (i > classIndex) {\r\n\t\t\t\t\tvector[i - 1] = value;\r\n\t\t\t\t} else {\r\n\t\t\t\t\tclassValue = value;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\treturn new Instance(vector, classValue);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Parses a sparse instance from strings.\r\n\t * \r\n\t * @param data the string array.\r\n\t * @param attrSet the attributes set.\r\n\t * @return a sparse instance from strings.\r\n\t */\r\n\tprivate static Instance parseSparseInstance(String[] data, TreeSet<Integer> attrSet) {\r\n\t\tdouble targetValue = Double.parseDouble(data[0]);\r\n\t\tint[] indices = new int[data.length - 1];\r\n\t\tdouble[] values = new double[data.length - 1];\r\n\t\tfor (int i = 0; i < indices.length; i++) {\r\n\t\t\tString[] pair = data[i + 1].split(\":\");\r\n\t\t\tindices[i] = Integer.parseInt(pair[0]);\r\n\t\t\tvalues[i] = Double.parseDouble(pair[1]);\r\n\t\t\tattrSet.add(indices[i]);\r\n\t\t}\r\n\t\treturn new Instance(indices, values, targetValue);\r\n\t}\r\n\r\n\t/**\r\n\t * Parses a sparse instance from strings.\r\n\t * \r\n\t * @param data the string array.\r\n\t * @return a sparse instance from strings.\r\n\t */\r\n\tprivate static Instance parseSparseInstance(String[] data) {\r\n\t\tdouble classValue = Double.parseDouble(data[0]);\r\n\t\tint[] indices = new int[data.length - 1];\r\n\t\tdouble[] values = new double[data.length - 1];\r\n\t\tfor (int i = 0; i < indices.length; i++) {\r\n\t\t\tString[] pair = data[i + 1].split(\":\");\r\n\t\t\tindices[i] = Integer.parseInt(pair[0]);\r\n\t\t\tvalues[i] = Double.parseDouble(pair[1]);\r\n\t\t}\r\n\t\treturn new Instance(indices, values, classValue);\r\n\t}\r\n\r\n\t/**\r\n\t * Assigns target attribute for a dataset.\r\n\t * \r\n\t * @param instances the data set.\r\n\t */\r\n\tprivate static void assignTargetAttribute(Instances instances) {\r\n\t\tboolean isInteger = true;\r\n\t\tfor (Instance instance : instances) {\r\n\t\t\tif (!MathUtils.isInteger(instance.getTarget())) {\r\n\t\t\t\tisInteger = false;\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t}\r\n\t\tif (isInteger) {\r\n\t\t\tTreeSet<Integer> set = new TreeSet<>();\r\n\t\t\tfor (Instance instance : instances) {\r\n\t\t\t\tdouble target = instance.getTarget();\r\n\t\t\t\tset.add((int) target);\r\n\t\t\t}\r\n\t\t\tString[] states = new String[set.size()];\r\n\t\t\tint i = 0;\r\n\t\t\tfor (Integer v : set) {\r\n\t\t\t\tstates[i++] = v.toString();\r\n\t\t\t}\r\n\t\t\tinstances.setTargetAttribute(new NominalAttribute(\"target\", states));\r\n\t\t} else {\r\n\t\t\tinstances.setTargetAttribute(new NumericalAttribute(\"target\"));\r\n\t\t}\r\n\t}\r\n\t\r\n\t/**\r\n\t * Parses double value from a string. Missing value is supported.\r\n\t * \r\n\t * @param s the string to parse.\r\n\t * @return double value.\r\n\t */\r\n\tprivate static double parseDouble(String s) {\r\n\t\tif (s.equals(\"?\")) {\r\n\t\t\treturn Double.NaN;\r\n\t\t} else {\r\n\t\t\treturn Double.parseDouble(s);\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/io/InstancesWriter.java",
    "content": "package mltk.core.io;\n\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\n\n/**\n * Class for writing instances.\n * \n * @author Yin Lou\n * \n */\npublic class InstancesWriter {\n\n\t/**\n\t * Writes a set of dense instances to attribute file and data file.\n\t * \n\t * @param instances the dense instances to write.\n\t * @param attFile the attribute file path.\n\t * @param dataFile the data file path.\n\t * @throws IOException\n\t */\n\tpublic static void write(Instances instances, String attFile, String dataFile) throws IOException {\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tPrintWriter out = new PrintWriter(attFile);\n\t\tfor (Attribute attribute : attributes) {\n\t\t\tout.println(attribute);\n\t\t}\n\t\tout.println(instances.getTargetAttribute() + \" (target)\");\n\t\tout.flush();\n\t\tout.close();\n\n\t\twrite(instances, dataFile);\n\t}\n\n\t/**\n\t * Writes a set of dense/sparse instances to data file.\n\t * \n\t * @param instances the dense instances to write.\n\t * @param file the data file path.\n\t * @throws IOException\n\t */\n\tpublic static void write(Instances instances, String file) throws IOException {\n\t\tPrintWriter out = new PrintWriter(file);\n\t\tfor (Instance instance : instances) {\n\t\t\tout.println(instance);\n\t\t}\n\t\tout.flush();\n\t\tout.close();\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/io/package-info.java",
    "content": "/**\n * Contains classes for reading and writing datasets.\n */\npackage mltk.core.io;"
  },
  {
    "path": "src/main/java/mltk/core/package-info.java",
    "content": "/**\n * Provides classes and interfaces for handling datasets and attributes.\n */\npackage mltk.core;"
  },
  {
    "path": "src/main/java/mltk/core/processor/Discretizer.java",
    "content": "package mltk.core.processor;\r\n\r\nimport java.util.ArrayList;\r\nimport java.util.Collections;\r\nimport java.util.List;\r\n\r\nimport mltk.cmdline.Argument;\r\nimport mltk.cmdline.CmdLineParser;\r\nimport mltk.core.Attribute;\r\nimport mltk.core.BinnedAttribute;\r\nimport mltk.core.Bins;\r\nimport mltk.core.Instance;\r\nimport mltk.core.Instances;\r\nimport mltk.core.Attribute.Type;\r\nimport mltk.core.io.AttributesReader;\r\nimport mltk.core.io.InstancesReader;\r\nimport mltk.core.io.InstancesWriter;\r\nimport mltk.util.Element;\r\nimport mltk.util.tuple.DoublePair;\r\n\r\n/**\r\n * Class for discretizers.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Discretizer {\r\n\t\r\n\tstatic class Options {\r\n\r\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\r\n\t\tString attPath = null;\r\n\t\t\r\n\t\t@Argument(name = \"-t\", description = \"training file path\")\r\n\t\tString trainPath = null;\r\n\r\n\t\t@Argument(name = \"-i\", description = \"input dataset path\", required = true)\r\n\t\tString inputPath = null;\r\n\r\n\t\t@Argument(name = \"-d\", description = \"discretized attribute file path\")\r\n\t\tString disAttPath = null;\r\n\r\n\t\t@Argument(name = \"-m\", description = \"output attribute file path\")\r\n\t\tString outputAttPath = null;\r\n\r\n\t\t@Argument(name = \"-o\", description = \"output dataset path\", required = true)\r\n\t\tString outputPath = null;\r\n\r\n\t\t@Argument(name = \"-n\", description = \"maximum num of bins (default: 256)\")\r\n\t\tint maxNumBins = 256;\r\n\r\n\t}\r\n\t\r\n\t/**\r\n\t * Discretizes datasets.\r\n\t * \r\n\t * <pre>\r\n\t * Usage: mltk.core.processor.Discretizer\r\n\t * -i\tinput dataset path\r\n\t * -o\toutput dataset path\r\n\t * [-r]\tattribute file path\r\n\t * [-t]\ttraining file path\r\n\t * [-d]\tdiscretized attribute file path\r\n\t * [-m]\toutput attribute file path\r\n\t * [-n]\tmaximum num of bins (default: 256)\r\n\t * </pre>\r\n\t * \r\n\t * @param args the command line arguments.\r\n\t * @throws Exception\r\n\t */\r\n\tpublic static void main(String[] args) throws Exception {\r\n\t\tOptions app = new Options();\r\n\t\tCmdLineParser parser = new CmdLineParser(Discretizer.class, app);\r\n\t\ttry {\r\n\t\t\tparser.parse(args);\r\n\t\t\tif (app.maxNumBins < 0) {\r\n\t\t\t\tthrow new IllegalArgumentException();\r\n\t\t\t}\r\n\t\t} catch (IllegalArgumentException e) {\r\n\t\t\tparser.printUsage();\r\n\t\t\tSystem.exit(1);\r\n\t\t}\r\n\t\tList<Attribute> attributes = null;\r\n\t\tif (app.trainPath != null) {\r\n\t\t\tInstances trainSet = InstancesReader.read(app.attPath, app.trainPath);\r\n\t\t\tattributes = trainSet.getAttributes();\r\n\t\t\tfor (int i = 0; i < attributes.size(); i++) {\r\n\t\t\t\tAttribute attribute = attributes.get(i);\r\n\t\t\t\tif (attribute.getType() == Type.NUMERIC) {\r\n\t\t\t\t\t// Only discretize numeric attributes\r\n\t\t\t\t\tDiscretizer.discretize(trainSet, i, app.maxNumBins);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t} else if (app.disAttPath != null) {\r\n\t\t\tattributes = AttributesReader.read(app.disAttPath).v1;\r\n\t\t} else {\r\n\t\t\tparser.printUsage();\r\n\t\t\tSystem.exit(1);\r\n\t\t}\r\n\r\n\t\tInstances instances = InstancesReader.read(app.attPath, app.inputPath);\r\n\t\tList<Attribute> attrs = instances.getAttributes();\r\n\t\tfor (int i = 0; i < attrs.size(); i++) {\r\n\t\t\tAttribute attr = attrs.get(i);\r\n\t\t\tif (attr.getType() == Type.NUMERIC) {\r\n\t\t\t\tBinnedAttribute binnedAttr = (BinnedAttribute) attributes.get(i);\r\n\t\t\t\t// Only discretize numeric attributes\r\n\t\t\t\tDiscretizer.discretize(instances, i, binnedAttr.getBins());\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tif (app.outputAttPath != null) {\r\n\t\t\tInstancesWriter.write(instances, app.outputAttPath, app.outputPath);\r\n\t\t} else {\r\n\t\t\tInstancesWriter.write(instances, app.outputPath);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Compute bins for a list of values.\r\n\t * \r\n\t * @param x the vector of input data.\r\n\t * @param maxNumBins the number of bins.\r\n\t * @return bins for a list of values.\r\n\t */\r\n\tpublic static Bins computeBins(double[] x, int maxNumBins) {\r\n\t\tList<Element<Double>> list = new ArrayList<>();\r\n\t\tfor (double v : x) {\r\n\t\t\tif (!Double.isNaN(v)) {\r\n\t\t\t\tlist.add(new Element<Double>(1.0, v));\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn computeBins(list, maxNumBins);\r\n\t}\r\n\r\n\t/**\r\n\t * Compute bins for a specified attribute.\r\n\t * \r\n\t * @param instances the dataset to discretize.\r\n\t * @param attIndex the attribute index.\r\n\t * @param maxNumBins the number of bins.\r\n\t * @return bins for a specified attribute.\r\n\t */\r\n\tpublic static Bins computeBins(Instances instances, int attIndex, int maxNumBins) {\r\n\t\tAttribute attribute = instances.getAttributes().get(attIndex);\r\n\t\tList<Element<Double>> list = new ArrayList<>();\r\n\t\tfor (Instance instance : instances) {\r\n\t\t\tif (!instance.isMissing(attribute.getIndex())) {\r\n\t\t\t\tlist.add(new Element<Double>(instance.getWeight(), instance.getValue(attribute)));\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn computeBins(list, maxNumBins);\r\n\t}\r\n\r\n\t/**\r\n\t * Compute bins for a list of values.\r\n\t * \r\n\t * @param list the histogram.\r\n\t * @param maxNumBins the number of bins.\r\n\t * @return bins for a list of values.\r\n\t */\r\n\tpublic static Bins computeBins(List<Element<Double>> list, int maxNumBins) {\r\n\t\tCollections.sort(list);\r\n\t\tList<DoublePair> stats = new ArrayList<>();\r\n\t\tgetStats(list, stats);\r\n\t\tif (stats.size() <= maxNumBins) {\r\n\t\t\tdouble[] a = new double[stats.size()];\r\n\t\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\t\ta[i] = stats.get(i).v1;\r\n\t\t\t}\r\n\t\t\treturn new Bins(a, a);\r\n\t\t} else {\r\n\t\t\tdouble totalWeight = 0;\r\n\t\t\tfor (DoublePair stat : stats) {\r\n\t\t\t\ttotalWeight += stat.v2;\r\n\t\t\t}\r\n\t\t\tdouble binSize = totalWeight / maxNumBins;\r\n\t\t\tList<Double> boundaryList = new ArrayList<>();\r\n\t\t\tList<Double> medianList = new ArrayList<>();\r\n\t\t\tint start = 0;\r\n\t\t\tdouble weight = 0;\r\n\t\t\tfor (int i = 0; i < stats.size(); i++) {\r\n\t\t\t\tweight += stats.get(i).v2;\r\n\t\t\t\ttotalWeight -= stats.get(i).v2;\r\n\t\t\t\tif (weight >= binSize) {\r\n\t\t\t\t\tif (i == start) {\r\n\t\t\t\t\t\tboundaryList.add(stats.get(start).v1);\r\n\t\t\t\t\t\tmedianList.add(stats.get(start).v1);\r\n\t\t\t\t\t\tweight = 0;\r\n\t\t\t\t\t\tstart = i + 1;\r\n\t\t\t\t\t} else {\r\n\t\t\t\t\t\tdouble d1 = weight - binSize;\r\n\t\t\t\t\t\tdouble d2 = stats.get(i).v2 - d1;\r\n\t\t\t\t\t\tif (d1 < d2) {\r\n\t\t\t\t\t\t\tboundaryList.add(stats.get(i).v1);\r\n\t\t\t\t\t\t\tmedianList.add(getMedian(stats, start, weight / 2));\r\n\t\t\t\t\t\t\tstart = i + 1;\r\n\t\t\t\t\t\t\tweight = 0;\r\n\t\t\t\t\t\t} else {\r\n\t\t\t\t\t\t\tweight -= stats.get(i).v2;\r\n\t\t\t\t\t\t\tboundaryList.add(stats.get(i - 1).v1);\r\n\t\t\t\t\t\t\tmedianList.add(getMedian(stats, start, weight / 2));\r\n\t\t\t\t\t\t\tstart = i;\r\n\t\t\t\t\t\t\tweight = stats.get(i).v2;\r\n\t\t\t\t\t\t}\r\n\t\t\t\t\t}\r\n\t\t\t\t\tbinSize = (totalWeight + weight) / (maxNumBins - boundaryList.size());\r\n\t\t\t\t} else if (i == stats.size() - 1) {\r\n\t\t\t\t\tboundaryList.add(stats.get(i).v1);\r\n\t\t\t\t\tmedianList.add(getMedian(stats, start, weight / 2));\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tdouble[] boundaries = new double[boundaryList.size()];\r\n\t\t\tdouble[] medians = new double[medianList.size()];\r\n\t\t\tfor (int i = 0; i < boundaries.length; i++) {\r\n\t\t\t\tboundaries[i] = boundaryList.get(i);\r\n\t\t\t\tmedians[i] = medianList.get(i);\r\n\t\t\t}\r\n\t\t\treturn new Bins(boundaries, medians);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Discretizes an attribute using bins.\r\n\t * \r\n\t * @param instances the dataset to discretize.\r\n\t * @param attIndex the attribute index.\r\n\t * @param bins the bins.\r\n\t */\r\n\tpublic static void discretize(Instances instances, int attIndex, Bins bins) {\r\n\t\tAttribute attribute = instances.getAttributes().get(attIndex);\r\n\t\tBinnedAttribute binnedAttribute = new BinnedAttribute(attribute.getName(), bins);\r\n\t\tbinnedAttribute.setIndex(attribute.getIndex());\r\n\t\tinstances.getAttributes().set(attIndex, binnedAttribute);\r\n\t\tfor (Instance instance : instances) {\r\n\t\t\tif (!instance.isMissing(attribute.getIndex())) {\r\n\t\t\t\tint v = bins.getIndex(instance.getValue(attribute.getIndex()));\r\n\t\t\t\tinstance.setValue(attribute.getIndex(), v);\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Discretized an attribute with specified number of bins.\r\n\t * \r\n\t * @param instances the dataset to discretize.\r\n\t * @param attIndex the attribute index.\r\n\t * @param maxNumBins the number of bins.\r\n\t */\r\n\tpublic static void discretize(Instances instances, int attIndex, int maxNumBins) {\r\n\t\tBins bins = computeBins(instances, attIndex, maxNumBins);\r\n\t\tdiscretize(instances, attIndex, bins);\r\n\t}\r\n\r\n\tstatic double getMedian(List<DoublePair> stats, int start, double midPoint) {\r\n\t\tdouble weight = 0;\r\n\t\tfor (int i = start; i < stats.size(); i++) {\r\n\t\t\tweight += stats.get(i).v2;\r\n\t\t\tif (weight >= midPoint) {\r\n\t\t\t\treturn stats.get(i).v1;\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn stats.get((start + stats.size()) / 2).v1;\r\n\t}\r\n\r\n\tstatic void getStats(List<Element<Double>> list, List<DoublePair> stats) {\r\n\t\tif (list.size() == 0) {\r\n\t\t\treturn;\r\n\t\t}\r\n\t\tdouble totalWeight = list.get(0).element;\r\n\t\tdouble lastValue = list.get(0).weight;\r\n\t\tfor (int i = 1; i < list.size(); i++) {\r\n\t\t\tElement<Double> element = list.get(i);\r\n\t\t\tdouble value = element.weight;\r\n\t\t\tdouble weight = element.element;\r\n\t\t\tif (value != lastValue) {\r\n\t\t\t\tstats.add(new DoublePair(lastValue, totalWeight));\r\n\t\t\t\tlastValue = value;\r\n\t\t\t\ttotalWeight = weight;\r\n\t\t\t} else {\r\n\t\t\t\ttotalWeight += weight;\r\n\t\t\t}\r\n\t\t}\r\n\t\tstats.add(new DoublePair(lastValue, totalWeight));\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic Discretizer() {\r\n\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/processor/InstancesSplitter.java",
    "content": "package mltk.core.processor;\r\n\r\nimport java.io.File;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\n\r\nimport mltk.cmdline.Argument;\r\nimport mltk.cmdline.CmdLineParser;\r\nimport mltk.core.Attribute;\r\nimport mltk.core.Attribute.Type;\r\nimport mltk.core.BinnedAttribute;\r\nimport mltk.core.Instance;\r\nimport mltk.core.Instances;\r\nimport mltk.core.NominalAttribute;\r\nimport mltk.core.io.InstancesReader;\r\nimport mltk.core.io.InstancesWriter;\r\nimport mltk.util.MathUtils;\r\nimport mltk.util.Random;\r\nimport mltk.util.StatUtils;\r\n\r\n/**\r\n * Class for cross validation.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class InstancesSplitter {\r\n\t\r\n\tstatic class Options {\r\n\r\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\r\n\t\tString attPath = null;\r\n\r\n\t\t@Argument(name = \"-i\", description = \"input dataset path\", required = true)\r\n\t\tString inputPath = null;\r\n\r\n\t\t@Argument(name = \"-o\", description = \"output directory path\", required = true)\r\n\t\tString outputDirPath = null;\r\n\r\n\t\t@Argument(name = \"-m\", description = \"splitting mode:parameter. Splitting mode can be split (s) and cross validation (c) (default: c:5)\")\r\n\t\tString crossValidationMode = \"c:5\";\r\n\t\t\r\n\t\t@Argument(name = \"-a\", description = \"attribute name to perform stratified sampling (default: null)\")\r\n\t\tString attToStrafity = null;\r\n\t\t\r\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\r\n\t\tlong seed = 0L;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Splits a dataset.\r\n\t * \r\n\t * <pre>\r\n\t * Usage: mltk.core.processor.InstancesSplitter\r\n\t * -i\tinput dataset path\r\n\t * -o\toutput directory path\r\n\t * [-r]\tattribute file path\r\n\t * [-m]\tsplitting mode:parameter. Splitting mode can be split (s) and cross validation (c) (default: c:5)\r\n\t * [-a]\tattribute name to perform stratified sampling (default: null)\r\n\t * [-s]\tseed of the random number generator (default: 0)\r\n\t * </pre>\r\n\t * \r\n\t * @param args the command line arguments.\r\n\t * @throws Exception\r\n\t */\r\n\tpublic static void main(String[] args) throws Exception {\r\n\t\tOptions opts = new Options();\r\n\t\tCmdLineParser parser = new CmdLineParser(InstancesSplitter.class, opts);\r\n\t\tString[] data = null;\r\n\t\ttry {\r\n\t\t\tparser.parse(args);\r\n\t\t\tdata = opts.crossValidationMode.split(\":\");\r\n\t\t\tif (data.length < 2) {\r\n\t\t\t\tthrow new IllegalArgumentException();\r\n\t\t\t}\r\n\t\t} catch (IllegalArgumentException e) {\r\n\t\t\tparser.printUsage();\r\n\t\t\tSystem.exit(1);\r\n\t\t}\r\n\r\n\t\tRandom.getInstance().setSeed(opts.seed);\r\n\r\n\t\tInstances instances = InstancesReader.read(opts.attPath, opts.inputPath);\r\n\r\n\t\tFile attFile = new File(opts.attPath);\r\n\t\tString prefix = attFile.getName().split(\"\\\\.\")[0];\r\n\r\n\t\tFile dir = new File(opts.outputDirPath);\r\n\t\tif (!dir.exists()) {\r\n\t\t\tdir.mkdirs();\r\n\t\t}\r\n\r\n\t\tswitch (data[0]) {\r\n\t\t\tcase \"c\":\r\n\t\t\t\tint k = Integer.parseInt(data[1]);\r\n\t\t\t\tif (data.length == 2) {\r\n\t\t\t\t\tInstances[][] folds = InstancesSplitter.createCrossValidationFolds(instances, opts.attToStrafity, k);\r\n\t\t\t\t\tfor (int i = 0; i < folds.length; i++) {\r\n\t\t\t\t\t\tString path = opts.outputDirPath + File.separator + \"cv.\" + i;\r\n\t\t\t\t\t\tFile directory = new File(path);\r\n\t\t\t\t\t\tif (!directory.exists()) {\r\n\t\t\t\t\t\t\tdirectory.mkdir();\r\n\t\t\t\t\t\t}\r\n\t\t\t\t\t\tInstancesWriter.write(folds[i][0], path + File.separator + prefix + \".attr\", path\r\n\t\t\t\t\t\t\t\t+ File.separator + prefix + \".train.all\");\r\n\t\t\t\t\t\tInstancesWriter.write(folds[i][1], path + File.separator + prefix + \".test\");\r\n\t\t\t\t\t}\r\n\t\t\t\t} else {\r\n\t\t\t\t\tdouble ratio = Double.parseDouble(data[2]);\r\n\t\t\t\t\tInstances[][] folds = InstancesSplitter.createCrossValidationFolds(instances, opts.attToStrafity, k, ratio);\r\n\t\t\t\t\tfor (int i = 0; i < folds.length; i++) {\r\n\t\t\t\t\t\tString path = opts.outputDirPath + File.separator + \"cv.\" + i;\r\n\t\t\t\t\t\tFile directory = new File(path);\r\n\t\t\t\t\t\tif (!directory.exists()) {\r\n\t\t\t\t\t\t\tdirectory.mkdir();\r\n\t\t\t\t\t\t}\r\n\t\t\t\t\t\tInstancesWriter.write(folds[i][0], path + File.separator + prefix + \".attr\", path\r\n\t\t\t\t\t\t\t\t+ File.separator + prefix + \".train\");\r\n\t\t\t\t\t\tInstancesWriter.write(folds[i][1], path + File.separator + prefix + \".valid\");\r\n\t\t\t\t\t\tInstancesWriter.write(folds[i][2], path + File.separator + prefix + \".test\");\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t\tbreak;\r\n\t\t\tcase \"s\":\r\n\t\t\t\tif (data.length == 2) {\r\n\t\t\t\t\tdouble ratio = Double.parseDouble(data[1]);\r\n\t\t\t\t\tInstances[] datasets = InstancesSplitter.split(instances, opts.attToStrafity, ratio);\r\n\t\t\t\t\tInstancesWriter.write(datasets[0], opts.outputDirPath + File.separator + prefix + \".attr\",\r\n\t\t\t\t\t\t\topts.outputDirPath + File.separator + prefix + \".train\");\r\n\t\t\t\t\tInstancesWriter.write(datasets[1], opts.outputDirPath + File.separator + prefix + \".valid\");\r\n\t\t\t\t} else if (data.length == 3) {\r\n\t\t\t\t\tdouble ratioTrain = Double.parseDouble(data[1]);\r\n\t\t\t\t\tdouble ratioValid = Double.parseDouble(data[2]);\r\n\t\t\t\t\tdouble[] ratios = new double[] {\r\n\t\t\t\t\t\t\tratioTrain,\r\n\t\t\t\t\t\t\tratioValid\r\n\t\t\t\t\t};\r\n\t\t\t\t\tInstances[] datasets = InstancesSplitter.split(instances, opts.attToStrafity, ratios);\r\n\t\t\t\t\tInstancesWriter.write(datasets[0], opts.outputDirPath + File.separator + prefix + \".attr\",\r\n\t\t\t\t\t\t\topts.outputDirPath + File.separator + prefix + \".train\");\r\n\t\t\t\t\tInstancesWriter.write(datasets[1], opts.outputDirPath + File.separator + prefix + \".valid\");\r\n\t\t\t\t} else if (data.length == 4) {\r\n\t\t\t\t\tdouble ratioTrain = Double.parseDouble(data[1]);\r\n\t\t\t\t\tdouble ratioValid = Double.parseDouble(data[2]);\r\n\t\t\t\t\tdouble ratioTest = Double.parseDouble(data[3]);\r\n\t\t\t\t\tdouble[] ratios = new double[] {\r\n\t\t\t\t\t\t\tratioTrain,\r\n\t\t\t\t\t\t\tratioValid,\r\n\t\t\t\t\t\t\tratioTest\r\n\t\t\t\t\t};\r\n\t\t\t\t\tInstances[] datasets = InstancesSplitter.split(instances, opts.attToStrafity, ratios);\r\n\t\t\t\t\tInstancesWriter.write(datasets[0], opts.outputDirPath + File.separator + prefix + \".attr\",\r\n\t\t\t\t\t\t\topts.outputDirPath + File.separator + prefix + \".train\");\r\n\t\t\t\t\tInstancesWriter.write(datasets[1], opts.outputDirPath + File.separator + prefix + \".valid\");\r\n\t\t\t\t\tInstancesWriter.write(datasets[2], opts.outputDirPath + File.separator + prefix + \".test\");\r\n\t\t\t\t} else {\r\n\t\t\t\t\tdouble[] ratios = new double[data.length - 1];\r\n\t\t\t\t\tfor (int i = 0; i < ratios.length; i++) {\r\n\t\t\t\t\t\tratios[i] = Double.parseDouble(data[i + 1]);\r\n\t\t\t\t\t}\r\n\t\t\t\t\tInstances[] datasets = InstancesSplitter.split(instances, opts.attToStrafity, ratios);\r\n\t\t\t\t\tfor (int i = 0; i < datasets.length; i++) {\r\n\t\t\t\t\t\tInstancesWriter.write(datasets[i], opts.outputDirPath + File.separator + prefix + \".data.\" + i);\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t\tbreak;\r\n\t\t\tdefault:\r\n\t\t\t\tbreak;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Creates cross validation folds from a dataset. For each cross validation fold contains a training set and a test\r\n\t * set.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param k the number of cross validation folds.\r\n\t * @return the cross validation datasets.\r\n\t */\r\n\tpublic static Instances[][] createCrossValidationFolds(Instances instances, int k) {\r\n\t\tInstances[] datasets = split(instances, k);\r\n\t\tInstances[][] folds = new Instances[k][2];\r\n\t\tfor (int i = 0; i < k; i++) {\r\n\t\t\tfolds[i][1] = datasets[i];\r\n\t\t\tfolds[i][0] = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t\tfor (int j = 0; j < k; j++) {\r\n\t\t\t\tif (i == j) {\r\n\t\t\t\t\tcontinue;\r\n\t\t\t\t}\r\n\t\t\t\tfor (Instance instance : datasets[j]) {\r\n\t\t\t\t\tfolds[i][0].add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn folds;\r\n\t}\r\n\r\n\t/**\r\n\t * Creates cross validation folds from a dataset. For each cross validation fold contains a training set, a\r\n\t * validation set and a test set.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param k the number of cross validation folds.\r\n\t * @param ratio the ratio that controls how many points in the training set for each fold.\r\n\t * @return the cross validation datasets.\r\n\t */\r\n\tpublic static Instances[][] createCrossValidationFolds(Instances instances, int k, double ratio) {\r\n\t\tInstances[] datasets = split(instances, k);\r\n\t\tInstances[][] folds = new Instances[k][3];\r\n\t\tfor (int i = 0; i < k; i++) {\r\n\t\t\tfolds[i][2] = datasets[i];\r\n\t\t\tInstances trainSet = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t\tfor (int j = 0; j < k; j++) {\r\n\t\t\t\tif (i == j) {\r\n\t\t\t\t\tcontinue;\r\n\t\t\t\t}\r\n\t\t\t\tfor (Instance instance : datasets[j]) {\r\n\t\t\t\t\ttrainSet.add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tInstances[] tmp = split(trainSet, ratio);\r\n\t\t\tfolds[i][0] = tmp[0];\r\n\t\t\tfolds[i][1] = tmp[1];\r\n\t\t}\r\n\t\treturn folds;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Creates cross validation folds from a dataset. For each cross validation fold contains a training set and a test\r\n\t * set.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param attToStratify the attribute to perform stratified sampling.\r\n\t * @param k the number of cross validation folds.\r\n\t * @return the cross validation datasets.\r\n\t */\r\n\tpublic static Instances[][] createCrossValidationFolds(Instances instances, String attToStratify, int k) {\r\n\t\tInstances[] datasets = split(instances, attToStratify, k);\r\n\t\tInstances[][] folds = new Instances[k][2];\r\n\t\tfor (int i = 0; i < k; i++) {\r\n\t\t\tfolds[i][1] = datasets[i];\r\n\t\t\tfolds[i][0] = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t\tfor (int j = 0; j < k; j++) {\r\n\t\t\t\tif (i == j) {\r\n\t\t\t\t\tcontinue;\r\n\t\t\t\t}\r\n\t\t\t\tfor (Instance instance : datasets[j]) {\r\n\t\t\t\t\tfolds[i][0].add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn folds;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Creates cross validation folds from a dataset. For each cross validation fold contains a training set, a\r\n\t * validation set and a test set.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param attToStratify the attribute to perform stratified sampling.\r\n\t * @param k the number of cross validation folds.\r\n\t * @param ratio the ratio that controls how many points in the training set for each fold.\r\n\t * @return the cross validation datasets.\r\n\t */\r\n\tpublic static Instances[][] createCrossValidationFolds(Instances instances, String attToStratify, int k, double ratio) {\r\n\t\tInstances[] datasets = split(instances, attToStratify, k);\r\n\t\tInstances[][] folds = new Instances[k][3];\r\n\t\tfor (int i = 0; i < k; i++) {\r\n\t\t\tfolds[i][2] = datasets[i];\r\n\t\t\tInstances trainSet = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t\tfor (int j = 0; j < k; j++) {\r\n\t\t\t\tif (i == j) {\r\n\t\t\t\t\tcontinue;\r\n\t\t\t\t}\r\n\t\t\t\tfor (Instance instance : datasets[j]) {\r\n\t\t\t\t\ttrainSet.add(instance);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tInstances[] tmp = split(trainSet, attToStratify, ratio);\r\n\t\t\tfolds[i][0] = tmp[0];\r\n\t\t\tfolds[i][1] = tmp[1];\r\n\t\t}\r\n\t\treturn folds;\r\n\t}\r\n\r\n\t/**\r\n\t * Splits the dataset according to the ratios. This method returns multiple instances objects, the size of each\r\n\t * partition is determined by the ratios array. The sum of ratios can be smaller than 1.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param ratios the ratios.\r\n\t * @return partitions of the dataset.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, double... ratios) {\r\n\t\tif (StatUtils.sum(ratios) > 1) {\r\n\t\t\tthrow new IllegalArgumentException(\"Sum of ratios is larger than 1\");\r\n\t\t}\r\n\t\tInstances dataset = new Instances(instances);\r\n\t\tdataset.shuffle(Random.getInstance().getRandom());\r\n\t\tInstances[] datasets = new Instances[ratios.length];\r\n\t\tfor (int i = 0; i < datasets.length; i++) {\r\n\t\t\tdatasets[i] = new Instances(dataset.getAttributes(), dataset.getTargetAttribute());\r\n\t\t}\r\n\t\tdouble sumRatios = StatUtils.sum(ratios);\r\n\t\tint n = 0;\r\n\t\tfor (int k = 0; k < datasets.length; k++) {\r\n\t\t\tint m = (int) (dataset.size() * ratios[k]);\r\n\t\t\tif (k == datasets.length - 1 && MathUtils.equals(sumRatios, 1.0)) {\r\n\t\t\t\tm = dataset.size() - n;\r\n\t\t\t}\r\n\t\t\tInstances partition = datasets[k];\r\n\t\t\tfor (int i = n; i < n + m; i++) {\r\n\t\t\t\tpartition.add(dataset.get(i));\r\n\t\t\t}\r\n\t\t\tn += m;\r\n\t\t}\r\n\t\treturn datasets;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Splits the dataset according to the ratio. This method returns two instances objects, the size of the first one\r\n\t * is 100% * ratio of the orignal dataset while the size of the second one is 100% * (1 - ratio) of the orignal\r\n\t * dataset.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param ratio the ratio.\r\n\t * @return two smaller datasets.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, double ratio) {\r\n\t\treturn split(instances, new double[] { ratio, 1 - ratio });\r\n\t}\r\n\t\r\n\t/**\r\n\t * Splits the dataset into k equi-sized datasets.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param k the number of datasets to return.\r\n\t * @return k equi-sized datasets.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, int k) {\r\n\t\tInstances dataset = new Instances(instances);\r\n\t\tdataset.shuffle(Random.getInstance().getRandom());\r\n\t\tInstances[] datasets = new Instances[k];\r\n\t\tfor (int i = 0; i < datasets.length; i++) {\r\n\t\t\tdatasets[i] = new Instances(dataset.getAttributes(), dataset.getTargetAttribute());\r\n\t\t}\r\n\t\tfor (int i = 0; i < dataset.size(); i++) {\r\n\t\t\tdatasets[i % datasets.length].add(dataset.get(i));\r\n\t\t}\r\n\t\treturn datasets;\r\n\t}\r\n\r\n\t/**\r\n\t * Splits the dataset according to the ratio. This method returns two instances objects, the size of the first one\r\n\t * is 100% * ratio of the orignal dataset while the size of the second one is 100% * (1 - ratio) of the orignal\r\n\t * dataset.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param attToStratify the attribute to perform stratified sampling.\r\n\t * @param ratio the ratio.\r\n\t * @return two smaller datasets.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, String attToStratify, double ratio) {\r\n\t\treturn split(instances, attToStratify, new double[] { ratio, 1 - ratio });\r\n\t}\r\n\r\n\t/**\r\n\t * Splits the dataset according to the ratios. This method returns multiple instances objects, the size of each\r\n\t * partition is determined by the ratios array. The sum of ratios can be smaller than 1.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param attToStratify the attribute to perform stratified sampling.\r\n\t * @param ratios the ratios.\r\n\t * @return partitions of the dataset.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, String attToStratify, double... ratios) {\r\n\t\tif (attToStratify == null) {\r\n\t\t\treturn split(instances, ratios);\r\n\t\t}\r\n\t\tList<List<Instance>> strata = getStrata(instances, attToStratify);\r\n\t\tif (strata == null) {\r\n\t\t\treturn split(instances, ratios);\r\n\t\t}\r\n\t\tInstances[] datasets = new Instances[ratios.length];\r\n\t\tfor (int i = 0; i < datasets.length; i++) {\r\n\t\t\tdatasets[i] = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t}\r\n\t\tdouble sumRatios = StatUtils.sum(ratios);\r\n\t\tfor (List<Instance> list : strata) {\r\n\t\t\tint n = 0;\r\n\t\t\tfor (int k = 0; k < datasets.length; k++) {\r\n\t\t\t\tint m = (int) (list.size() * ratios[k]);\r\n\t\t\t\tif (k == datasets.length -1 && MathUtils.equals(sumRatios, 1.0)) {\r\n\t\t\t\t\tm = list.size() - n;\r\n\t\t\t\t}\r\n\t\t\t\tInstances partition = datasets[k];\r\n\t\t\t\tfor (int i = n; i < n + m; i++) {\r\n\t\t\t\t\tpartition.add(list.get(i));\r\n\t\t\t\t}\r\n\t\t\t\tn += m;\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn datasets;\r\n\t}\r\n\r\n\t/**\r\n\t * Splits the dataset into k equi-sized datasets.\r\n\t * \r\n\t * @param instances the dataset.\r\n\t * @param attToStratify the attribute to perform stratified sampling.\r\n\t * @param k the number of datasets to return.\r\n\t * @return k equi-sized datasets.\r\n\t */\r\n\tpublic static Instances[] split(Instances instances, String attToStratify, int k) {\r\n\t\tif (attToStratify == null) {\r\n\t\t\treturn split(instances, k);\r\n\t\t}\r\n\t\tList<List<Instance>> strata = getStrata(instances, attToStratify);\r\n\t\tif (strata == null) {\r\n\t\t\treturn split(instances, k);\r\n\t\t}\r\n\t\tInstances[] datasets = new Instances[k];\r\n\t\tfor (int i = 0; i < datasets.length; i++) {\r\n\t\t\tdatasets[i] = new Instances(instances.getAttributes(), instances.getTargetAttribute());\r\n\t\t}\r\n\t\tfor (List<Instance> stratum : strata) {\r\n\t\t\tfor (int i = 0; i < stratum.size(); i++) {\r\n\t\t\t\tdatasets[i % datasets.length].add(stratum.get(i));\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn datasets;\r\n\t}\r\n\t\r\n\tprivate static List<List<Instance>> getStrata(Instances instances, String attToStratify) {\r\n\t\tList<List<Instance>> lists = new ArrayList<>();\r\n\t\tInstances dataset = new Instances(instances);\r\n\t\tdataset.shuffle(Random.getInstance().getRandom());\r\n\t\tif (instances.getTargetAttribute().getName().equals(attToStratify)) {\r\n\t\t\tAttribute targetAtt = instances.getTargetAttribute();\r\n\t\t\tif (targetAtt == null || targetAtt.getType() == Type.NUMERIC) {\r\n\t\t\t\treturn null;\r\n\t\t\t}\r\n\t\t\tint cardinality = 0;\r\n\t\t\tif (targetAtt.getType() == Type.BINNED) {\r\n\t\t\t\tcardinality = ((BinnedAttribute) targetAtt).getNumBins();\r\n\t\t\t} else {\r\n\t\t\t\tcardinality = ((NominalAttribute) targetAtt).getCardinality();\r\n\t\t\t}\r\n\t\t\tfor (int i = 0; i < cardinality; i++) {\r\n\t\t\t\tlists.add(new ArrayList<Instance>());\r\n\t\t\t}\r\n\t\t\tfor (Instance instance : instances) {\r\n\t\t\t\tint idx = (int) instance.getTarget();\r\n\t\t\t\tlists.get(idx).add(instance);\r\n\t\t\t}\r\n\t\t} else {\r\n\t\t\tList<Attribute> attributes = instances.getAttributes();\r\n\t\t\tAttribute attr = null;\r\n\t\t\tfor (Attribute att : attributes) {\r\n\t\t\t\tif (att.getName().equals(attToStratify)) {\r\n\t\t\t\t\tattr = att;\r\n\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tif (attr == null || attr.getType() == Type.NUMERIC) {\r\n\t\t\t\treturn null;\r\n\t\t\t}\r\n\t\t\tint cardinality = 0;\r\n\t\t\tif (attr.getType() == Type.BINNED) {\r\n\t\t\t\tcardinality = ((BinnedAttribute) attr).getNumBins();\r\n\t\t\t} else {\r\n\t\t\t\tcardinality = ((NominalAttribute) attr).getCardinality();\r\n\t\t\t}\r\n\t\t\tfor (int i = 0; i < cardinality; i++) {\r\n\t\t\t\tlists.add(new ArrayList<Instance>());\r\n\t\t\t}\r\n\t\t\tfor (Instance instance : dataset) {\r\n\t\t\t\tint idx = (int) instance.getValue(attr);\r\n\t\t\t\tlists.get(idx).add(instance);\r\n\t\t\t}\r\n\t\t}\r\n\t\t\r\n\t\treturn lists;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/core/processor/OneHotEncoder.java",
    "content": "package mltk.core.processor;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.NumericalAttribute;\n\n/**\n * Class for one-hot encoders. Binned attributes and nominal attributes are\n * transformed into a set of binary attributes using one-hot encoding. \n * \n * @author Yin Lou\n *\n */\npublic class OneHotEncoder {\n\n\t/**\n\t * Transforms all binned and nominal attributes to binary attributes using\n\t * one-hot encoding.\n\t * \n\t * @param instances the input instances.\n\t * @return the transformed instances.\n\t */\n\tpublic Instances process(Instances instances) {\n\t\tList<Attribute> attrListOld = instances.getAttributes();\n\t\tList<Attribute> attrListNew = new ArrayList<>();\n\t\tint[] offset = new int[instances.dimension()];\n\t\tboolean[] isNumerical = new boolean[instances.dimension()];\n\t\tint attIndex = 0;\n\t\tfor (int j = 0; j < attrListOld.size(); j++) {\n\t\t\tAttribute attribute = attrListOld.get(j);\n\t\t\toffset[j] = attIndex;\n\t\t\tString name = attribute.getName();\n\t\t\tif (attribute instanceof BinnedAttribute) {\n\t\t\t\tBinnedAttribute binnedAttribute = (BinnedAttribute) attribute;\n\t\t\t\tint size = binnedAttribute.getNumBins();\n\t\t\t\tfor (int k = 0; k < size; k++) {\n\t\t\t\t\tNumericalAttribute attr = new NumericalAttribute(name + \"_\" + k);\n\t\t\t\t\tattr.setIndex(attIndex++);\n\t\t\t\t\tattrListNew.add(attr);\n\t\t\t\t}\n\t\t\t} else if (attribute instanceof NominalAttribute) {\n\t\t\t\tNominalAttribute nominalAttribute = (NominalAttribute) attribute;\n\t\t\t\tString[] states = nominalAttribute.getStates();\n\t\t\t\tfor (String state : states) {\n\t\t\t\t\tNumericalAttribute attr = new NumericalAttribute(name + \"_\" + state);\n\t\t\t\t\tattr.setIndex(attIndex++);\n\t\t\t\t\tattrListNew.add(attr);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tNumericalAttribute attr = new NumericalAttribute(name);\n\t\t\t\tattr.setIndex(attIndex++);\n\t\t\t\tattrListNew.add(attr);\n\t\t\t\tisNumerical[j] = true;\n\t\t\t}\n\t\t}\n\t\t\n\t\tInstances instancesNew = new Instances(attrListNew, instances.getTargetAttribute(), \n\t\t\t\tinstances.size());\n\t\tfor (Instance instance : instances) {\n\t\t\tint[] indices = new int[instances.dimension()];\n\t\t\tdouble[] values = new double[instances.dimension()];\n\t\t\tfor (int j = 0; j < attrListOld.size(); j++) {\n\t\t\t\tif (isNumerical[j]) {\n\t\t\t\t\tindices[j] = offset[j];\n\t\t\t\t\tvalues[j] = instance.getValue(attrListOld.get(j));\n\t\t\t\t} else {\n\t\t\t\t\tint v = (int) instance.getValue(attrListOld.get(j));\n\t\t\t\t\tindices[j] = offset[j] + v;\n\t\t\t\t\tvalues[j] = 1.0;\n\t\t\t\t}\n\t\t\t}\n\t\t\tInstance instanceNew = new Instance(indices, values, instance.getTarget(), \n\t\t\t\t\tinstance.getWeight());\n\t\t\tinstancesNew.add(instanceNew);\t\n\t\t}\n\t\t\n\t\treturn instancesNew;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/core/processor/package-info.java",
    "content": "/**\n * Provides classes for processing datasets.\n */\npackage mltk.core.processor;"
  },
  {
    "path": "src/main/java/mltk/feature/selection/BackwardElimination.java",
    "content": "package mltk.feature.selection;\r\n\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\n\r\nimport mltk.core.Attribute;\r\nimport mltk.core.Instances;\r\nimport mltk.predictor.BaggedEnsembleLearner;\r\nimport mltk.predictor.Regressor;\r\nimport mltk.predictor.evaluation.Evaluator;\r\nimport mltk.predictor.tree.ensemble.ag.AdditiveGrovesLearner;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.tuple.DoublePair;\r\nimport mltk.util.tuple.Pair;\r\n\r\n/**\r\n * Class for feature selection using backward elimination.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BackwardElimination {\r\n\r\n\t/**\r\n\t * Selects features using backward elimination.\r\n\t * \r\n\t * @param trainSet the training set.\r\n\t * @param validSet the validation set.\r\n\t * @param learner the learner to use.\r\n\t * @param numIters the number of iterations to estimate the mean and std for full complexity models.\r\n\t * @return the list of selected features and &lt;mean, std$gt; pair for full complexity models.\r\n\t */\r\n\tpublic static Pair<List<Attribute>, DoublePair> select(Instances trainSet, Instances validSet,\r\n\t\t\tBaggedEnsembleLearner learner, int numIters) {\r\n\t\tList<Attribute> attributes = trainSet.getAttributes();\r\n\t\tList<Attribute> selected = new ArrayList<>(attributes);\r\n\t\tDoublePair perf = null;\r\n\t\tfor (;;) {\r\n\t\t\tif (selected.size() == 0) {\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t\tboolean changed = false;\r\n\t\t\ttrainSet.setAttributes(selected);\r\n\t\t\tperf = evaluateModel(trainSet, validSet, learner, numIters);\r\n\t\t\tSystem.out.println(\"Mean: \" + perf.v1 + \" Std: \" + perf.v2);\r\n\t\t\tint i;\r\n\t\t\tfor (i = 0; i < selected.size();) {\r\n\t\t\t\tList<Attribute> attList = new ArrayList<>(selected);\r\n\t\t\t\tAttribute attr = attList.get(i);\r\n\t\t\t\tattList.remove(i);\r\n\t\t\t\ttrainSet.setAttributes(attList);\r\n\t\t\t\tRegressor regressor = (Regressor) learner.build(trainSet);\r\n\t\t\t\tdouble rmse = Evaluator.evalRMSE(regressor, validSet);\r\n\t\t\t\tSystem.out.println(\"Testing: \" + attr.getName() + \" RMSE: \" + rmse);\r\n\t\t\t\tif (perf.v1 - perf.v2 * 3 <= rmse && rmse <= perf.v1 + perf.v2 * 3) {\r\n\t\t\t\t\t// Eliminate feature\r\n\t\t\t\t\tselected.remove(i);\r\n\t\t\t\t\tchanged = true;\r\n\t\t\t\t\tSystem.out.println(\"Eliminate: \" + attr.getName());\r\n\t\t\t\t} else {\r\n\t\t\t\t\ti++;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tif (!changed) {\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t}\r\n\t\ttrainSet.setAttributes(attributes);\r\n\t\treturn new Pair<List<Attribute>, DoublePair>(selected, perf);\r\n\t}\r\n\r\n\t/**\r\n\t * Selects features using backward elimination in Additive Groves.\r\n\t * \r\n\t * @param trainSet the training set.\r\n\t * @param validSet the validation set.\r\n\t * @param learner the learner to use.\r\n\t * @param baggingIters the number of bagging iterations.\r\n\t * @param numTrees the number of trees in a grove.\r\n\t * @param alpha the alpha.\r\n\t * @param numIters the number of iterations to estimate the mean and std for full complexity models.\r\n\t * @return the list of selected features and $lt;mean, std$gt; pair for full complexity models.\r\n\t */\r\n\tpublic static Pair<List<Attribute>, DoublePair> select(Instances trainSet, Instances validSet,\r\n\t\t\tAdditiveGrovesLearner learner, int baggingIters, int numTrees, double alpha, int numIters) {\r\n\t\tList<Attribute> attributes = trainSet.getAttributes();\r\n\t\tList<Attribute> selected = new ArrayList<>(attributes);\r\n\t\tDoublePair perf = null;\r\n\t\tfor (;;) {\r\n\t\t\tif (selected.size() == 0) {\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t\tboolean changed = false;\r\n\t\t\ttrainSet.setAttributes(selected);\r\n\t\t\tperf = evaluateModel(trainSet, validSet, learner, baggingIters, numTrees, alpha, numIters);\r\n\t\t\tSystem.out.println(\"Mean: \" + perf.v1 + \" Std: \" + perf.v2);\r\n\t\t\tint i;\r\n\t\t\tfor (i = 0; i < selected.size();) {\r\n\t\t\t\tList<Attribute> attList = new ArrayList<>(selected);\r\n\t\t\t\tAttribute attr = attList.get(i);\r\n\t\t\t\tSystem.out.println(\"Testing: \" + attr.getName());\r\n\t\t\t\tattList.remove(i);\r\n\t\t\t\ttrainSet.setAttributes(attList);\r\n\t\t\t\tRegressor regressor = learner.runLayeredTraining(trainSet,baggingIters, numTrees, alpha);\r\n\t\t\t\tdouble rmse = Evaluator.evalRMSE(regressor, validSet);\r\n\t\t\t\tSystem.out.println(\"Testing: \" + attr.getName() + \" RMSE: \" + rmse);\r\n\t\t\t\tif (perf.v1 - perf.v2 * 3 <= rmse && rmse <= perf.v1 + perf.v2 * 3) {\r\n\t\t\t\t\t// Eliminate feature\r\n\t\t\t\t\tselected.remove(i);\r\n\t\t\t\t\tchanged = true;\r\n\t\t\t\t\tSystem.out.println(\"Eliminate: \" + attr.getName());\r\n\t\t\t\t} else {\r\n\t\t\t\t\ti++;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tif (!changed) {\r\n\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t}\r\n\t\ttrainSet.setAttributes(attributes);\r\n\t\treturn new Pair<List<Attribute>, DoublePair>(selected, perf);\r\n\t}\r\n\r\n\tprivate static DoublePair evaluateModel(Instances trainSet, Instances validSet, BaggedEnsembleLearner learner,\r\n\t\t\tint numIters) {\r\n\t\t// Estimating std of full complexity model\r\n\t\tdouble[] rmse = new double[numIters];\r\n\t\tfor (int i = 0; i < rmse.length; i++) {\r\n\t\t\tRegressor regressor = (Regressor) learner.build(trainSet);\r\n\t\t\trmse[i] = Evaluator.evalRMSE(regressor, validSet);\r\n\t\t}\r\n\t\tdouble mean = StatUtils.mean(rmse);\r\n\t\tdouble std = StatUtils.sd(rmse);\r\n\t\treturn new DoublePair(mean, std);\r\n\t}\r\n\r\n\tprivate static DoublePair evaluateModel(Instances trainSet, Instances validSet, AdditiveGrovesLearner learner,\r\n\t\t\tint baggingIters, int numTrees, double alpha, int numIters) {\r\n\t\t// Estimating std of full complexity model\r\n\t\tdouble[] rmse = new double[numIters];\r\n\t\tfor (int i = 0; i < rmse.length; i++) {\r\n\t\t\tRegressor regressor = learner.runLayeredTraining(trainSet, baggingIters, numTrees, alpha);\r\n\t\t\trmse[i] = Evaluator.evalRMSE(regressor, validSet);\r\n\t\t\tSystem.out.println(\"\\tEvaluating model \" + (i + 1) + \" / \" + numIters + \"\\t\" + rmse[i]);\r\n\t\t}\r\n\t\tdouble mean = StatUtils.mean(rmse);\r\n\t\tdouble std = StatUtils.sd(rmse);\r\n\t\treturn new DoublePair(mean, std);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/feature/selection/package-info.java",
    "content": "/**\r\n * Contains classes for feature selection.\r\n */\r\npackage mltk.feature.selection;"
  },
  {
    "path": "src/main/java/mltk/predictor/BaggedEnsemble.java",
    "content": "package mltk.predictor;\r\n\r\nimport java.util.HashMap;\r\nimport java.util.Map;\r\n\r\nimport mltk.core.Instance;\r\n\r\n/**\r\n * Class for bagged ensembles.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BaggedEnsemble extends Ensemble {\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic BaggedEnsemble() {\r\n\t\tsuper();\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param capacity the capacity of this bagged ensemble.\r\n\t */\r\n\tpublic BaggedEnsemble(int capacity) {\r\n\t\tsuper(capacity);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\tif (predictors.size() == 0) {\r\n\t\t\treturn 0.0;\r\n\t\t} else {\r\n\t\t\tdouble prediction = 0.0;\r\n\t\t\tfor (Predictor predictor : predictors) {\r\n\t\t\t\tRegressor regressor = (Regressor) predictor;\r\n\t\t\t\tprediction += regressor.regress(instance);\r\n\t\t\t}\r\n\t\t\treturn prediction / predictors.size();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int classify(Instance instance) {\r\n\t\tif (predictors.size() == 0) {\r\n\t\t\t// Default: return first class\r\n\t\t\treturn 0;\r\n\t\t} else {\r\n\t\t\tMap<Integer, Integer> votes = new HashMap<>();\r\n\t\t\tfor (Predictor predictor : predictors) {\r\n\t\t\t\tClassifier classifier = (Classifier) predictor;\r\n\t\t\t\tint cls = (int) classifier.classify(instance);\r\n\t\t\t\tif (!votes.containsKey(cls)) {\r\n\t\t\t\t\tvotes.put(cls, 0);\r\n\t\t\t\t}\r\n\t\t\t\tvotes.put(cls, votes.get(cls) + 1);\r\n\t\t\t}\r\n\t\t\tint prediction = 0;\r\n\t\t\tint maxVotes = 0;\r\n\t\t\tfor (int cls : votes.keySet()) {\r\n\t\t\t\tint numVotes = votes.get(cls);\r\n\t\t\t\tif (numVotes > maxVotes) {\r\n\t\t\t\t\tmaxVotes = numVotes;\r\n\t\t\t\t\tprediction = cls;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\treturn prediction;\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BaggedEnsemble copy() {\r\n\t\tBaggedEnsemble copy = new BaggedEnsemble(predictors.size());\r\n\t\tfor (Predictor predictor : predictors) {\r\n\t\t\tcopy.add(predictor.copy());\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/BaggedEnsembleLearner.java",
    "content": "package mltk.predictor;\r\n\r\nimport mltk.core.Instances;\r\nimport mltk.core.Sampling;\r\n\r\n/**\r\n * Class for learning bagged ensembles.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BaggedEnsembleLearner extends Learner {\r\n\r\n\tprotected int baggingIters;\r\n\tprotected Learner learner;\r\n\tprotected Instances[] bags;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param baggingIters the number of bagging iterations.\r\n\t * @param learner the learner.\r\n\t */\r\n\tpublic BaggedEnsembleLearner(int baggingIters, Learner learner) {\r\n\t\tthis.baggingIters = baggingIters;\r\n\t\tthis.learner = learner;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the number of bagging iterations.\r\n\t * \r\n\t * @return the number of bagging iterations.\r\n\t */\r\n\tpublic int getBaggingIterations() {\r\n\t\treturn baggingIters;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the number of bagging iterations.\r\n\t * \r\n\t * @param baggingIters the number of bagging iterations.\r\n\t */\r\n\tpublic void setBaggingIterations(int baggingIters) {\r\n\t\tthis.baggingIters = baggingIters;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the learner.\r\n\t * \r\n\t * @return the learner.\r\n\t */\r\n\tpublic Learner getLearner() {\r\n\t\treturn learner;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the learner.\r\n\t * \r\n\t * @param learner the learner.\r\n\t */\r\n\tpublic void setLearner(Learner learner) {\r\n\t\tthis.learner = learner;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the bootstrap samples.\r\n\t * \r\n\t * @return the bootstrap samples.\r\n\t */\r\n\tpublic Instances[] getBags() {\r\n\t\treturn bags;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the bootstrap samples.\r\n\t * \r\n\t * @param bags the bootstrap samples.\r\n\t */\r\n\tpublic void setBags(Instances[] bags) {\r\n\t\tthis.bags = bags;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BaggedEnsemble build(Instances instances) {\r\n\t\t// Create bags\r\n\t\tbags = Sampling.createBags(instances, baggingIters);\r\n\r\n\t\tBaggedEnsemble baggedEnsemble = new BaggedEnsemble(bags.length);\r\n\t\tfor (Instances bag : bags) {\r\n\t\t\tbaggedEnsemble.add(learner.build(bag));\r\n\t\t}\r\n\t\treturn baggedEnsemble;\r\n\t}\r\n\r\n\t/**\r\n\t * Builds a bagged ensemble.\r\n\t * \r\n\t * @param bags the bootstrap samples.\r\n\t * @return a bagged ensemble.\r\n\t */\r\n\tpublic BaggedEnsemble build(Instances[] bags) {\r\n\t\tBaggedEnsemble baggedEnsemble = new BaggedEnsemble(bags.length);\r\n\t\tfor (Instances bag : bags) {\r\n\t\t\tbaggedEnsemble.add(learner.build(bag));\r\n\t\t}\r\n\t\treturn baggedEnsemble;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/BoostedEnsemble.java",
    "content": "package mltk.predictor;\r\n\r\nimport mltk.core.Instance;\r\n\r\n/**\r\n * Class for boosted ensembles.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BoostedEnsemble extends Ensemble {\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic BoostedEnsemble() {\r\n\t\tsuper();\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param capacity the capacity of the boosted ensemble.\r\n\t */\r\n\tpublic BoostedEnsemble(int capacity) {\r\n\t\tsuper(capacity);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\tdouble prediction = 0.0;\r\n\t\tfor (Predictor predictor : predictors) {\r\n\t\t\tRegressor regressor = (Regressor) predictor;\r\n\t\t\tprediction += regressor.regress(instance);\r\n\t\t}\r\n\t\treturn prediction;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int classify(Instance instance) {\r\n\t\tdouble pred = regress(instance);\r\n\t\tif (pred >= 0) {\r\n\t\t\treturn 1;\r\n\t\t} else {\r\n\t\t\treturn -1;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Removes a particular predictor.\r\n\t * \r\n\t * @param index the index of the predictor to remove.\r\n\t */\r\n\tpublic void remove(int index) {\r\n\t\tpredictors.remove(index);\r\n\t}\r\n\r\n\t/**\r\n\t * Removes the last predictor.\r\n\t */\r\n\tpublic void removeLast() {\r\n\t\tif (predictors.size() > 0) {\r\n\t\t\tpredictors.remove(predictors.size() - 1);\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BoostedEnsemble copy() {\r\n\t\tBoostedEnsemble copy = new BoostedEnsemble(predictors.size());\r\n\t\tfor (Predictor predictor : predictors) {\r\n\t\t\tcopy.add(predictor.copy());\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Classifier.java",
    "content": "package mltk.predictor;\r\n\r\nimport mltk.core.Instance;\r\n\r\n/**\r\n * Interface for classfiers.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic interface Classifier extends Predictor {\r\n\r\n\t/**\r\n\t * Classifies an instance.\r\n\t * \r\n\t * @param instance the instance to classify.\r\n\t * @return a classified value.\r\n\t */\r\n\tpublic int classify(Instance instance);\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Ensemble.java",
    "content": "package mltk.predictor;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\n\r\n/**\r\n * Abstract class for ensembles.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic abstract class Ensemble implements Classifier, Regressor {\r\n\r\n\tprotected List<Predictor> predictors;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic Ensemble() {\r\n\t\tpredictors = new ArrayList<>();\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param capacity the capacity of this ensemble.\r\n\t */\r\n\tpublic Ensemble(int capacity) {\r\n\t\tpredictors = new ArrayList<>(capacity);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns a particular predictor.\r\n\t * \r\n\t * @param index the index of predictor.\r\n\t * @return a particular predictor.\r\n\t */\r\n\tpublic Predictor get(int index) {\r\n\t\treturn predictors.get(index);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the internal predictors.\r\n\t * \r\n\t * @return the internal predictors.\r\n\t */\r\n\tpublic List<Predictor> getPredictors() {\r\n\t\treturn predictors;\r\n\t}\r\n\r\n\t/**\r\n\t * Adds a new predictor to the ensemble.\r\n\t * \r\n\t * @param predictor the new predictor.\r\n\t */\r\n\tpublic void add(Predictor predictor) {\r\n\t\tpredictors.add(predictor);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the size of this ensemble.\r\n\t * \r\n\t * @return the size of this ensemble.\r\n\t */\r\n\tpublic int size() {\r\n\t\treturn predictors.size();\r\n\t}\r\n\r\n\t/**\r\n\t * Clears this ensemble.\r\n\t */\r\n\tpublic void clear() {\r\n\t\tpredictors.clear();\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tint capacity = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\tpredictors = new ArrayList<>(capacity);\r\n\t\tin.readLine();\r\n\t\tfor (int i = 0; i < capacity; i++) {\r\n\t\t\tString line = in.readLine();\r\n\t\t\tString predictorName = line.substring(1, line.length() - 1).split(\": \")[1];\r\n\t\t\tClass<?> clazz = Class.forName(predictorName);\r\n\t\t\tPredictor predictor = (Predictor) clazz.getDeclaredConstructor().newInstance();\r\n\t\t\tpredictor.read(in);\r\n\t\t\tpredictors.add(predictor);\r\n\t\t\tin.readLine();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"Ensemble: \" + predictors.size());\r\n\t\tout.println();\r\n\t\tfor (Predictor predictor : predictors) {\r\n\t\t\tpredictor.write(out);\r\n\t\t\tout.println();\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Family.java",
    "content": "package mltk.predictor;\n\n/**\n * Class for response distribution family. This class is used for GLMs/GAMs.\n * \n * @author Yin Lou\n *\n */\npublic enum Family {\n\n\tGAUSSIAN(\"gaussian\", LinkFunction.IDENTITY),\n\tBINOMIAL(\"binomial\", LinkFunction.LOGIT);\n\n\t/**\n\t * Parses an enumeration from a string.\n\t * \n\t * @param name the family name.\n\t * @return a parsed distribution.\n\t */\n\tpublic static Family get(String name) {\n\t\tfor (Family family : Family.values()) {\n\t\t\tif (name.startsWith(family.name)) {\n\t\t\t\treturn family;\n\t\t\t}\n\t\t}\n\t\tthrow new IllegalArgumentException(\"Invalid family name: \" + name);\n\t}\n\tString name;\n\t\n\tLinkFunction link;\n\t\n\tFamily(String name, LinkFunction link) {\n\t\tthis.name = name;\n\t\tthis.link = link;\n\t}\n\n\t/**\n\t * Returns the default link function for this family.\n\t * \n\t * @return the default link function for this family.\n\t */\n\tpublic LinkFunction getDefaultLinkFunction() {\n\t\treturn link;\n\t}\n\n\t/**\n\t * Returns the string representation of this family with default link function.\n\t */\n\tpublic String toString() {\n\t\treturn name + \"(\" + link + \")\";\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/HoldoutValidatedLearner.java",
    "content": "package mltk.predictor;\n\nimport mltk.core.Instances;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.Metric;\n\n/**\n * Class for holdout validated learners.\n * \n * @author Yin Lou\n *\n */\npublic abstract class HoldoutValidatedLearner extends Learner {\n\n\tprotected Instances validSet;\n\tprotected Metric metric;\n\tprotected ConvergenceTester ct;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic HoldoutValidatedLearner() {\n\t\tct = new ConvergenceTester(-1, 0, 1.0);\n\t}\n\t\n\t/**\n\t * Returns the validation set.\n\t * \n\t * @return the validation set.\n\t */\n\tpublic Instances getValidSet() {\n\t\treturn validSet;\n\t}\n\n\t/**\n\t * Sets the validation set.\n\t * \n\t * @param validSet the validation set.\n\t */\n\tpublic void setValidSet(Instances validSet) {\n\t\tthis.validSet = validSet;\n\t}\n\t\n\t/**\n\t * Returns the metric.\n\t * \n\t * @return the metric.\n\t */\n\tpublic Metric getMetric() {\n\t\treturn metric;\n\t}\n\n\t/**\n\t * Sets the metric. \n\t * \n\t * @param metric the metric.\n\t */\n\tpublic void setMetric(Metric metric) {\n\t\tthis.metric = metric;\n\t}\n\t\n\t/**\n\t * Returns the convergence tester.\n\t * \n\t * @return the convergence tester.\n\t */\n\tpublic ConvergenceTester getConvergenceTester() {\n\t\treturn ct;\n\t}\n\t\n\t/**\n\t * Sets the convergence tester.\n\t * \n\t * @param ct the convergence tester to set.\n\t */\n\tpublic void setConvergenceTester(ConvergenceTester ct) {\n\t\tthis.ct = ct;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Learner.java",
    "content": "package mltk.predictor;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TreeMap;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.SparseVector;\nimport mltk.predictor.evaluation.Error;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.RMSE;\nimport mltk.util.MathUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\nimport mltk.util.tuple.IntDoublePair;\n\n/**\n * Class for learners.\n * \n * @author Yin Lou\n * \n */\npublic abstract class Learner {\n\t\n\tprotected boolean verbose;\n\n\t/**\n\t * Returns {@code true} if we output something during the training.\n\t * \n\t * @return {@code true} if we output something during the training.\n\t */\n\tpublic boolean isVerbose() {\n\t\treturn verbose;\n\t}\n\n\t/**\n\t * Sets whether we output something during the training.\n\t * \n\t * @param verbose the switch if we output things during training.\n\t */\n\tpublic void setVerbose(boolean verbose) {\n\t\tthis.verbose = verbose;\n\t}\n\t\n\t/**\n\t * Enumeration of learning tasks.\n\t * \n\t */\n\tpublic enum Task {\n\n\t\t/**\n\t\t * Classification task.\n\t\t */\n\t\tCLASSIFICATION(\"classification\"),\n\t\t/**\n\t\t * Regression task.\n\t\t */\n\t\tREGRESSION(\"regression\");\n\n\t\tString task;\n\n\t\tTask(String task) {\n\t\t\tthis.task = task;\n\t\t}\n\n\t\t/**\n\t\t * Returns the string representation of learning tasks.\n\t\t */\n\t\tpublic String toString() {\n\t\t\treturn task;\n\t\t}\n\n\t\t/**\n\t\t * Parses a task from a string.\n\t\t * \n\t\t * @param name the name of the task.\n\t\t * @return a parsed task.\n\t\t */\n\t\tpublic static Task get(String name) {\n\t\t\tfor (Task task : Task.values()) {\n\t\t\t\tif (task.task.startsWith(name)) {\n\t\t\t\t\treturn task;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new IllegalArgumentException(\"Invalid task name: \" + name);\n\t\t}\n\t\t\n\t\t/**\n\t\t * Returns the default metric for this task.\n\t\t * \n\t\t * @return the default metric for this task.\n\t\t */\n\t\tpublic Metric getDefaultMetric() {\n\t\t\tMetric metric = null;\n\t\t\tswitch (this) {\n\t\t\t\tcase CLASSIFICATION:\n\t\t\t\t\tmetric = new Error();\n\t\t\t\t\tbreak;\n\t\t\t\tcase REGRESSION:\n\t\t\t\t\tmetric = new RMSE();\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t\treturn metric;\n\t\t}\n\n\t}\n\n\t/**\n\t * Builds a predictor from training set.\n\t * \n\t * @param instances the training set.\n\t * @return a predictior.\n\t */\n\tpublic abstract Predictor build(Instances instances);\n\n\t/**\n\t * Returns {@code true} if the instances are treated as sparse.\n\t * \n\t * @param instances the instances to test.\n\t * @return {@code true} if the instances are treated as sparse.\n\t */\n\tprotected boolean isSparse(Instances instances) {\n\t\tint numSparseInstances = 0;\n\t\tfor (Instance instance : instances) {\n\t\t\tif (instance.isSparse()) {\n\t\t\t\tnumSparseInstances++;\n\t\t\t}\n\t\t}\n\t\treturn numSparseInstances > instances.size() / 2;\n\t}\n\n\t/**\n\t * Returns the column-oriented format of sparse dataset. This method automatically removes attributes with\n\t * close-to-zero variance.\n\t * \n\t * @param instances the instances.\n\t * @param normalize {@code true} if all the columns are normalized.\n\t * @return the column-oriented format of sparse dataset.\n\t */\n\tprotected SparseDataset getSparseDataset(Instances instances, boolean normalize) {\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tint maxAttrId = attributes.get(attributes.size() - 1).getIndex();\n\t\tboolean[] included = new boolean[maxAttrId + 1];\n\t\tfor (Attribute attribute : attributes) {\n\t\t\tincluded[attribute.getIndex()] = true;\n\t\t}\n\n\t\tfinal int n = instances.size();\n\t\tMap<Integer, List<IntDoublePair>> map = new TreeMap<>();\n\t\tdouble[] y = new double[n];\n\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tSparseVector vector = (SparseVector) instance.getVector();\n\t\t\tint[] indices = vector.getIndices();\n\t\t\tdouble[] values = vector.getValues();\n\t\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\t\tif (included[indices[j]]) {\n\t\t\t\t\tif (!map.containsKey(indices[j])) {\n\t\t\t\t\t\tmap.put(indices[j], new ArrayList<IntDoublePair>());\n\t\t\t\t\t}\n\t\t\t\t\tList<IntDoublePair> list = map.get(indices[j]);\n\t\t\t\t\tlist.add(new IntDoublePair(i, values[j]));\n\t\t\t\t}\n\t\t\t}\n\t\t\ty[i] = instance.getTarget();\n\t\t}\n\n\t\tList<Integer> attrsList = new ArrayList<>(map.size());\n\t\tList<int[]> indicesList = new ArrayList<>(map.size());\n\t\tList<double[]> valuesList = new ArrayList<>(map.size());\n\t\tList<Double> stdList = new ArrayList<>(map.size());\n\t\tList<Double> cList = null;\n\t\tif (normalize) {\n\t\t\tcList = new ArrayList<>();\n\t\t}\n\t\tdouble factor = Math.sqrt(n);\n\t\tfor (Map.Entry<Integer, List<IntDoublePair>> entry : map.entrySet()) {\n\t\t\tInteger attr = entry.getKey();\n\t\t\tList<IntDoublePair> list = entry.getValue();\n\t\t\tint[] indices = new int[list.size()];\n\t\t\tdouble[] values = new double[list.size()];\n\t\t\tfor (int i = 0; i < list.size(); i++) {\n\t\t\t\tIntDoublePair pair = list.get(i);\n\t\t\t\tindices[i] = pair.v1;\n\t\t\t\tvalues[i] = pair.v2;\n\t\t\t}\n\t\t\tdouble std = StatUtils.sd(values, n);\n\t\t\tif (std > MathUtils.EPSILON) {\n\t\t\t\tattrsList.add(attr);\n\t\t\t\tindicesList.add(indices);\n\t\t\t\tvaluesList.add(values);\n\t\t\t\tstdList.add(std);\n\t\t\t\tif (normalize) {\n\t\t\t\t\t// Normalize the data\n\t\t\t\t\tdouble c = factor / std;\n\t\t\t\t\tVectorUtils.multiply(values, c);\n\t\t\t\t\tcList.add(c);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfinal int p = attrsList.size();\n\t\tint[] attrs = new int[p];\n\t\tint[][] indices = new int[p][];\n\t\tdouble[][] values = new double[p][];\n\t\tfor (int j = 0; j < p; j++) {\n\t\t\tattrs[j] = attrsList.get(j);\n\t\t\tindices[j] = indicesList.get(j);\n\t\t\tvalues[j] = valuesList.get(j);\n\t\t}\n\t\t\n\t\tdouble[] std = new double[stdList.size()];\n\t\tfor (int j = 0; j < std.length; j++) {\n\t\t\tstd[j] = stdList.get(j);\n\t\t}\n\t\tdouble[] c = null;\n\t\tif (cList != null) {\n\t\t\tc = new double[cList.size()];\n\t\t\tfor (int j = 0; j < c.length; j++) {\n\t\t\t\tc[j] = cList.get(j);\n\t\t\t}\n\t\t}\n\n\t\treturn new SparseDataset(attrs, indices, values, y, std, c);\n\t}\n\n\t/**\n\t * Returns the column-oriented format of dense dataset. This method automatically removes attributes with\n\t * close-to-zero variance.\n\t * \n\t * @param instances the instances.\n\t * @param normalize {@code true} if all the columns are normalized.\n\t * @return the column-oriented format of dense dataset.\n\t */\n\tprotected DenseDataset getDenseDataset(Instances instances, boolean normalize) {\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tfinal int p = instances.dimension();\n\t\tfinal int n = instances.size();\n\n\t\t// Convert to column oriented format\n\t\tList<double[]> xList = new ArrayList<>(p);\n\t\tdouble[] y = new double[n];\n\t\tfor (int i = 0; i < n; i++) {\n\t\t\ty[i] = instances.get(i).getTarget();\n\t\t}\n\n\t\tList<Integer> attrsList = new ArrayList<>(p);\n\t\tList<Double> stdList = new ArrayList<>(p);\n\t\tList<Double> cList = null;\n\t\tif (normalize) {\n\t\t\tcList = new ArrayList<>();\n\t\t}\n\t\tdouble factor = Math.sqrt(n);\n\t\tfor (int j = 0; j < p; j++) {\n\t\t\tint attIndex = attributes.get(j).getIndex();\n\t\t\tdouble[] x = new double[n];\n\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\tx[i] = instances.get(i).getValue(attIndex);\n\t\t\t}\n\t\t\tdouble std = StatUtils.sd(x);\n\t\t\tif (std > MathUtils.EPSILON) {\n\t\t\t\tattrsList.add(attIndex);\n\t\t\t\txList.add(x);\n\t\t\t\tstdList.add(std);\n\t\t\t\tif (normalize) {\n\t\t\t\t\t// Normalize the data\n\t\t\t\t\tdouble c = factor / std;\n\t\t\t\t\tVectorUtils.multiply(x, c);\n\t\t\t\t\tcList.add(c);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tint[] attrs = new int[attrsList.size()];\n\t\tdouble[][] x = new double[attrsList.size()][];\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tattrs[j] = attrsList.get(j);\n\t\t\tx[j] = xList.get(j);\n\t\t}\n\t\t\n\t\tdouble[] std = new double[stdList.size()];\n\t\tfor (int j = 0; j < std.length; j++) {\n\t\t\tstd[j] = stdList.get(j);\n\t\t}\n\t\tdouble[] c = null;\n\t\tif (cList != null) {\n\t\t\tc = new double[cList.size()];\n\t\t\tfor (int j = 0; j < c.length; j++) {\n\t\t\t\tc[j] = cList.get(j);\n\t\t\t}\n\t\t}\n\n\t\treturn new DenseDataset(attrs, x, y, std, c);\n\t}\n\n\t/**\n\t * Class for sparse dataset.\n\t *\n\t */\n\tprotected class SparseDataset {\n\n\t\tpublic int[] attrs;\n\t\tpublic int[][] indices;\n\t\tpublic double[][] values;\n\t\tpublic double[] y;\n\t\tpublic double[] stdList;\n\t\tpublic double[] cList;\n\n\t\tSparseDataset(int[] attrs, int[][] indices, double[][] values, double[] y, double[] stdList,\n\t\t\t\tdouble[] cList) {\n\t\t\tthis.attrs = attrs;\n\t\t\tthis.indices = indices;\n\t\t\tthis.values = values;\n\t\t\tthis.y = y;\n\t\t\tthis.stdList = stdList;\n\t\t\tthis.cList = cList;\n\t\t}\n\n\t}\n\n\t/**\n\t * Class for dense dataset.\n\t *\n\t */\n\tprotected class DenseDataset {\n\n\t\tpublic int[] attrs;\n\t\tpublic double[][] x;\n\t\tpublic double[] y;\n\t\tpublic double[] stdList;\n\t\tpublic double[] cList;\n\n\t\tDenseDataset(int[] attrs, double[][] x, double[] y, double[] stdList, double[] cList) {\n\t\t\tthis.attrs = attrs;\n\t\t\tthis.x = x;\n\t\t\tthis.y = y;\n\t\t\tthis.stdList = stdList;\n\t\t\tthis.cList = cList;\n\t\t}\n\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/LinkFunction.java",
    "content": "package mltk.predictor;\n\nimport mltk.util.MathUtils;\n\n/**\n * Class for link functions.\n * \n * @author Yin Lou\n *\n */\npublic enum LinkFunction {\n\t\n\tIDENTITY(\"identity\"),\n\tLOGIT(\"logit\");\n\t\n\t/**\n\t * Parses a link function from a string.\n\t * \n\t * @param name the name of the function.\n\t * @return a parsed link function.\n\t */\n\tpublic static LinkFunction get(String name) {\n\t\tfor (LinkFunction link : LinkFunction.values()) {\n\t\t\tif (link.name.startsWith(name)) {\n\t\t\t\treturn link;\n\t\t\t}\n\t\t}\n\t\tthrow new IllegalArgumentException(\"Unknown link function: \" + name);\n\t}\n\t\n\tString name;\n\t\n\tLinkFunction(String name) {\n\t\tthis.name = name;\n\t}\n\t\n\t/**\n\t * Applies the inverse of this link function.\n\t * \n\t * @param x the argument.\n\t * @return the inverse of this link function.\n\t */\n\tpublic double applyInverse(double x) {\n\t\tdouble r = 0;\n\t\tswitch (this) {\n\t\t\tcase IDENTITY:\n\t\t\t\tr = x;\n\t\t\t\tbreak;\n\t\t\tcase LOGIT:\n\t\t\t\tr = MathUtils.sigmoid(x);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn r;\n\t}\n\t\n\t/**\n\t * Returns the string representation of this link function.\n\t */\n\tpublic String toString() {\n\t\treturn name;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Predictor.java",
    "content": "package mltk.predictor;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\nimport mltk.core.Copyable;\nimport mltk.core.Writable;\n\n/**\n * Interface for predictors.\n * \n * @author Yin Lou\n * \n */\npublic interface Predictor extends Writable, Copyable<Predictor> {\n\t\n\t/**\n\t * Reads in this predictor. This method is used in {@link mltk.predictor.io.PredictorReader}.\n\t * \n\t * @param in the reader.\n\t * @throws Exception\n\t */\n\tpublic void read(BufferedReader in) throws Exception;\n\n\t/**\n\t * Writes this predictor. This method is used in {@link mltk.predictor.io.PredictorWriter}.\n\t * \n\t * @param out the writer.\n\t * @throws Exception\n\t */\n\tpublic void write(PrintWriter out) throws Exception;\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/ProbabilisticClassifier.java",
    "content": "package mltk.predictor;\r\n\r\nimport mltk.core.Instance;\r\n\r\n/**\r\n * Interface for classifiers that predicts the class probabilities.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic interface ProbabilisticClassifier extends Classifier {\r\n\r\n\t/**\r\n\t * Returns the class probabilities.\r\n\t * \r\n\t * @param instance the instance to predict.\r\n\t * @return the class probabilities.\r\n\t */\r\n\tpublic double[] predictProbabilities(Instance instance);\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/Regressor.java",
    "content": "package mltk.predictor;\r\n\r\nimport mltk.core.Instance;\r\n\r\n/**\r\n * Interface for regressors.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic interface Regressor extends Predictor {\r\n\r\n\t/**\r\n\t * Regresses an instance.\r\n\t * \r\n\t * @param instance the instance to regress.\r\n\t * @return a regressed value.\r\n\t */\r\n\tpublic double regress(Instance instance);\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/AUC.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.util.Arrays;\nimport java.util.Comparator;\n\nimport mltk.core.Instances;\nimport mltk.util.tuple.DoublePair;\n\n/**\n * Class for evaluating area under ROC curve.\n * \n * @author Yin Lou\n *\n */\npublic class AUC extends SimpleMetric {\n\n\tprivate class DoublePairComparator implements Comparator<DoublePair> {\n\n\t\t@Override\n\t\tpublic int compare(DoublePair o1, DoublePair o2) {\n\t\t\tint cmp = Double.compare(o1.v1, o2.v1);\n\t\t\tif (cmp == 0) {\n\t\t\t\tcmp = Double.compare(o1.v2, o2.v2);\n\t\t\t}\n\t\t\treturn cmp;\n\t\t}\n\n\t}\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic AUC() {\n\t\tsuper(true);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\tDoublePair[] a = new DoublePair[preds.length];\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\ta[i] = new DoublePair(preds[i], targets[i]);\n\t\t}\n\t\treturn eval(a);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tDoublePair[] a = new DoublePair[preds.length];\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\ta[i] = new DoublePair(preds[i], instances.get(i).getTarget());\n\t\t}\n\t\treturn eval(a);\n\t}\n\t\n\tprotected double eval(DoublePair[] a) {\n\t\tArrays.sort(a, new DoublePairComparator());\n\t\tdouble[] fraction = new double[a.length];\n\t\tfor (int idx = 0; idx < fraction.length;) {\n\t\t\tint begin = idx;\n\t\t\tdouble pos = 0;\n\t\t\tfor (; idx < fraction.length && a[idx].v1 == a[begin].v1; idx++) {\n\t\t\t\tpos += a[idx].v2;\n\t\t\t}\n\t\t\tdouble frac = pos / (idx - begin);\n\t\t\tfor (int i = begin; i < idx; i++) {\n\t\t\t\tfraction[i] = frac;\n\t\t\t}\n\t\t}\n\n\t\tdouble tt = 0;\n\t\tdouble tf = 0;\n\t\tdouble ft = 0;\n\t\tdouble ff = 0;\n\n\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\ttf += a[i].v2;\n\t\t\tff += 1 - a[i].v2;\n\t\t}\n\n\t\tdouble area = 0;\n\t\tdouble tpfPrev = 0;\n\t\tdouble fpfPrev = 0;\n\n\t\tfor (int i = a.length - 1; i >= 0; i--) {\n\t\t\ttt += fraction[i];\n\t\t\ttf -= fraction[i];\n\t\t\tft += 1 - fraction[i];\n\t\t\tff -= 1 - fraction[i];\n\t\t\tdouble tpf = tt / (tt + tf);\n\t\t\tdouble fpf = 1.0 - ff / (ft + ff);\n\t\t\tarea += 0.5 * (tpf + tpfPrev) * (fpf - fpfPrev);\n\t\t\ttpfPrev = tpf;\n\t\t\tfpfPrev = fpf;\n\t\t}\n\n\t\treturn area;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/ConvergenceTester.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * Class for testing convergence given a list of metric values.\n * \n * @author Yin Lou\n *\n */\npublic class ConvergenceTester {\n\n\tprotected int minNumPoints;\n\tprotected int n;\n\tprotected double c;\n\tprotected double bestSoFar;\n\tprotected int bestIdx;\n\tprotected Metric metric;\n\tprotected List<Double> measureList;\n\t\n\t/**\n\t * Parses the convergence criteria string.\n\t * \n\t * @param cc the convergence criteria string.\n\t * @return a convergence tester.\n\t */\n\tpublic static ConvergenceTester parse(String cc) {\n\t\tint minNumPoints = -1;\n\t\tint n = 0;\n\t\tdouble c = 1.0;\n\t\tif (cc != null && !cc.equals(\"\")) {\n\t\t\tString[] strs = cc.split(\":\");\n\t\t\tif (strs.length > 0) {\n\t\t\t\tminNumPoints = Integer.parseInt(strs[0]);\n\t\t\t}\n\t\t\tif (strs.length > 1) {\n\t\t\t\tn = Integer.parseInt(strs[1]);\n\t\t\t}\n\t\t\tif (strs.length > 2) {\n\t\t\t\tc = Double.parseDouble(strs[2]);\n\t\t\t}\n\t\t}\n\t\treturn new ConvergenceTester(minNumPoints, n, c);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param minNumPoints the minimum number of points to be considered convergence.\n\t * @param c a constant factor in [0, 1].\n\t */\n\tpublic ConvergenceTester(int minNumPoints, double c) {\n\t\tthis(minNumPoints, 0, c, 1000);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param minNumPoints the minimum number of points to be considered convergence.\n\t * @param n the n.\n\t */\n\tpublic ConvergenceTester(int minNumPoints, int n) {\n\t\tthis(minNumPoints, n, 1.0, 1000);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param minNumPoints the minimum number of points to be considered convergence.\n\t * @param n the n.\n\t * @param c a constant factor in [0, 1].\n\t */\n\tpublic ConvergenceTester(int minNumPoints, int n, double c) {\n\t\tthis(minNumPoints, n, c, 1000);\n\t}\n\t\n\t/**\n\t * Constructor. A list of metric values is viewed as converged if the list\n\t * has at least {@code minNumPoints} and the {@code getBestIndex() + n < size() * c}.\n\t * \n\t * @param minNumPoints the minimum number of points to be considered convergence.\n\t * @param n the n.\n\t * @param c a constant factor in [0, 1].\n\t * @param capacity the initial capacity.\n\t */\n\tpublic ConvergenceTester(int minNumPoints, int n, double c, int capacity) {\n\t\tif (n < 0) {\n\t\t\tthrow new IllegalArgumentException(\"n has to be non-negative.\");\n\t\t}\n\t\tif (!(c >= 0 && c <= 1)) {\n\t\t\tthrow new IllegalArgumentException(\"c should to be in [0, 1].\");\n\t\t}\n\t\tthis.minNumPoints = minNumPoints;\n\t\tthis.n = n;\n\t\tthis.c = c;\n\t\tmeasureList = new ArrayList<>(capacity);\n\t}\n\t\n\t/**\n\t * Returns the metric.\n\t * \n\t * @return the metric.\n\t */\n\tpublic Metric getMetric() {\n\t\treturn metric;\n\t}\n\t\n\t/**\n\t * Sets the metric. This method also resets internal status of this tester.\n\t * \n\t * @param metric the metric to set.\n\t */\n\tpublic void setMetric(Metric metric) {\n\t\tthis.metric = metric;\n\t\tmeasureList.clear();\n\t\tbestSoFar = metric.worstValue();\n\t\tbestIdx = -1;\n\t}\n\t\n\t/**\n\t * Adds a measure.\n\t * \n\t * @param measure the metric value to add.\n\t */\n\tpublic void add(double measure) {\n\t\tmeasureList.add(measure);\n\t\t\n\t\tif (metric.isFirstBetter(measure, bestSoFar)) {\n\t\t\tbestSoFar = measure;\n\t\t\tbestIdx = measureList.size() - 1;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the index of best metric value so far.\n\t * \n\t * @return the index of best metric value so far.\n\t */\n\tpublic int getBestIndex() {\n\t\treturn bestIdx;\n\t}\n\t\n\t/**\n\t * Returns the best measure value so far.\n\t * \n\t * @return the best measure value so far.\n\t */\n\tpublic double getBestMetricValue() {\n\t\treturn bestSoFar;\n\t}\n\t\n\t/**\n\t * Returns the number of points.\n\t * \n\t * @return the number of points.\n\t */\n\tpublic int size() {\n\t\treturn measureList.size();\n\t}\n\t\n\t/**\n\t * Returns the list of metric values.\n\t * \n\t * @return the list of metric values.\n\t */\n\tpublic List<Double> getMeasureList() {\n\t\treturn measureList;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if the series is converged.\n\t * \n\t * @return {@code true} if the series is converged.\n\t */\n\tpublic boolean isConverged() {\n\t\treturn minNumPoints >= 0 && measureList.size() >= minNumPoints\n\t\t\t\t&& bestIdx > 0 && bestIdx + n < measureList.size() * c;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/Error.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.core.Instances;\n\n/**\n * Class for evaluating error rate.\n * \n * @author Yin Lou\n *\n */\npublic class Error extends SimpleMetric {\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Error() {\n\t\tsuper(false);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\tdouble error = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\t// Handles both probability and predicted label\n\t\t\tdouble cls = preds[i] <= 0 ? 0 : 1;\n\t\t\tif (cls != targets[i]) {\n\t\t\t\terror++;\n\t\t\t}\n\t\t}\n\t\treturn error / preds.length;\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tdouble error = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\t// Handles both probability and predicted label\n\t\t\tdouble cls = preds[i] <= 0 ? 0 : 1;\n\t\t\tif (cls != instances.get(i).getTarget()) {\n\t\t\t\terror++;\n\t\t\t}\n\t\t}\n\t\treturn error / preds.length;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/Evaluator.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.util.List;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.ProbabilisticClassifier;\nimport mltk.predictor.Classifier;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.OptimUtils;\n\n/**\n * Class for making evaluations.\n * \n * @author Yin Lou\n * \n */\npublic class Evaluator {\n\n\t/**\n\t * Returns the area under ROC curve.\n\t * \n\t * @param classifier a classifier that outputs probability.\n\t * @param instances the instances.\n\t * @return the area under ROC curve.\n\t */\n\tpublic static double evalAreaUnderROC(ProbabilisticClassifier classifier, Instances instances) {\n\t\tdouble[] probs = new double[instances.size()];\n\t\tdouble[] targets = new double[instances.size()];\n\t\tfor (int i = 0; i < probs.length; i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tprobs[i] = classifier.predictProbabilities(instance)[1];\n\t\t\ttargets[i] = instance.getTarget();\n\t\t}\n\t\treturn new AUC().eval(probs, targets);\n\t}\n\n\t/**\n\t * Returns the root mean squared error.\n\t * \n\t * @param preds the predictions.\n\t * @param targets the targets.\n\t * @return the root mean squared error.\n\t */\n\tpublic static double evalRMSE(List<Double> preds, List<Double> targets) {\n\t\tdouble rmse = 0;\n\t\tfor (int i = 0; i < preds.size(); i++) {\n\t\t\tdouble d = targets.get(i) - preds.get(i);\n\t\t\trmse += d * d;\n\t\t}\n\t\trmse = Math.sqrt(rmse / preds.size());\n\t\treturn rmse;\n\t}\n\n\t/**\n\t * Returns the root mean squared error.\n\t * \n\t * @param regressor the regressor.\n\t * @param instances the instances.\n\t * @return the root mean squared error.\n\t */\n\tpublic static double evalRMSE(Regressor regressor, Instances instances) {\n\t\tdouble rmse = 0;\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tdouble target = instance.getTarget();\n\t\t\tdouble pred = regressor.regress(instance);\n\t\t\tdouble d = target - pred;\n\t\t\trmse += d * d;\n\t\t}\n\t\trmse = Math.sqrt(rmse / instances.size());\n\t\treturn rmse;\n\t}\n\n\t/**\n\t * Returns the classification error.\n\t * \n\t * @param classifier the classifier.\n\t * @param instances the instances.\n\t * @return the classification error.\n\t */\n\tpublic static double evalError(Classifier classifier, Instances instances) {\n\t\tdouble error = 0;\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tdouble target = instance.getTarget();\n\t\t\tdouble pred = classifier.classify(instance);\n\t\t\tif (target != pred) {\n\t\t\t\terror++;\n\t\t\t}\n\t\t}\n\t\terror /= instances.size();\n\t\treturn error;\n\t}\n\t\n\t/**\n\t * Returns the logistic loss.\n\t * \n\t * @param regressor the regressor.\n\t * @param instances the instances.\n\t * @return the logistic loss.\n\t */\n\tpublic static double evalLogisticLoss(Regressor regressor, Instances instances) {\n\t\tdouble loss = 0;\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tdouble pred = regressor.regress(instance);\n\t\t\tloss += OptimUtils.computeLogisticLoss(pred, instance.getTarget());\n\t\t}\n\t\tloss /= instances.size();\n\t\treturn loss;\n\t}\n\t\n\t/**\n\t * Returns the mean absolute error.\n\t * \n\t * @param regressor the regressor.\n\t * @param instances the instances.\n\t * @return the mean absolute error.\n\t */\n\tpublic static double evalMAE(Regressor regressor, Instances instances) {\n\t\tdouble mae = 0;\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\tdouble target = instance.getTarget();\n\t\t\tdouble pred = regressor.regress(instance);\n\t\t\tdouble d = target - pred;\n\t\t\tmae += Math.abs(d);\n\t\t}\n\t\tmae /= instances.size();\n\t\treturn mae;\n\t}\n\n\tstatic class Options {\n\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\n\t\tString attPath = null;\n\n\t\t@Argument(name = \"-d\", description = \"data set path\", required = true)\n\t\tString dataPath = null;\n\n\t\t@Argument(name = \"-m\", description = \"model path\", required = true)\n\t\tString modelPath = null;\n\n\t\t@Argument(name = \"-e\", description = \"AUC (a), Error (c), Logistic Loss (l), MAE(m), RMSE (r) (default: r)\")\n\t\tString task = \"r\";\n\n\t}\n\n\t/**\n\t * Evaluates a predictor.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.evaluation.Evaluator\n\t * -d\tdata set path\n\t * -m\tmodel path\n\t * [-r]\tattribute file path\n\t * [-e]\tAUC (a), Error (c), Logistic Loss (l), MAE(m), RMSE (r) (default: r)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(Evaluator.class, opts);\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tInstances instances = InstancesReader.read(opts.attPath, opts.dataPath);\n\t\tmltk.predictor.Predictor predictor = PredictorReader.read(opts.modelPath);\n\n\t\tswitch (opts.task) {\n\t\t\tcase \"a\":\n\t\t\t\tdouble auc = evalAreaUnderROC((ProbabilisticClassifier) predictor, instances);\n\t\t\t\tSystem.out.println(\"AUC: \" + auc);\n\t\t\t\tbreak;\n\t\t\tcase \"c\":\n\t\t\t\tdouble error = evalError((Classifier) predictor, instances);\n\t\t\t\tSystem.out.println(\"Error: \" + error);\n\t\t\t\tbreak;\n\t\t\tcase \"l\":\n\t\t\t\tdouble logisticLoss = evalLogisticLoss((Regressor) predictor, instances);\n\t\t\t\tSystem.out.println(\"Logistic Loss: \" + logisticLoss);\n\t\t\t\tbreak;\n\t\t\tcase \"m\":\n\t\t\t\tdouble mae = evalMAE((Regressor) predictor, instances);\n\t\t\t\tSystem.out.println(\"MAE: \" + mae);\n\t\t\t\tbreak;\n\t\t\tcase \"r\":\n\t\t\t\tdouble rmse = evalRMSE((Regressor) predictor, instances);\n\t\t\t\tSystem.out.println(\"RMSE: \" + rmse);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/LogLoss.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.core.Instances;\nimport mltk.util.OptimUtils;\n\n/**\n * Class for evaluating log loss (cross entropy).\n * \n * @author Yin Lou\n *\n */\npublic class LogLoss extends SimpleMetric {\n\t\n\tprotected boolean isRawScore;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LogLoss() {\n\t\tthis(false);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param isRawScore {@code true} if raw score is expected as input.\n\t */\n\tpublic LogLoss(boolean isRawScore) {\n\t\tsuper(false);\n\t\tthis.isRawScore = isRawScore;\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\treturn OptimUtils.computeLogLoss(preds, targets, isRawScore);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tdouble logLoss = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tlogLoss += OptimUtils.computeLogLoss(preds[i], instances.get(i).getTarget(), isRawScore);\n\t\t}\n\t\tlogLoss /= preds.length;\n\t\treturn logLoss;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if raw score is expected as input.\n\t * \n\t * @return {@code true} if raw score is expected as input.\n\t */\n\tpublic boolean isRawScore() {\n\t\treturn isRawScore;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/LogisticLoss.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.core.Instances;\nimport mltk.util.OptimUtils;\n\n/**\n * Class for evaluating logistic loss.\n * \n * @author Yin Lou\n *\n */\npublic class LogisticLoss extends SimpleMetric {\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LogisticLoss() {\n\t\tsuper(false);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\treturn OptimUtils.computeLogisticLoss(preds, targets);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tdouble logisticLoss = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tlogisticLoss += OptimUtils.computeLogisticLoss(preds[i], instances.get(i).getTarget());\n\t\t}\n\t\tlogisticLoss /= preds.length;\n\t\treturn logisticLoss;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/MAE.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.core.Instances;\n\n/**\n * Class for evaluating mean absolute error (MAE).\n * \n * @author Yin Lou\n *\n */\npublic class MAE extends SimpleMetric {\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic MAE() {\n\t\tsuper(false);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\tdouble mae = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tmae += Math.abs(targets[i] - preds[i]);\n\t\t}\n\t\tmae /= preds.length;\n\t\treturn mae;\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tdouble mae = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tmae += Math.abs(instances.get(i).getTarget() - preds[i]);\n\t\t}\n\t\tmae /= preds.length;\n\t\treturn mae;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/Metric.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.util.List;\n\nimport mltk.core.Instances;\nimport mltk.util.MathUtils;\n\n/**\n * Class for evaluation metrics.\n * \n * @author Yin Lou\n *\n */\npublic abstract class Metric {\n\n\tprivate boolean isLargerBetter;\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param isLargerBetter {@code true} if larger value is better.\n\t */\n\tpublic Metric(boolean isLargerBetter) {\n\t\tthis.isLargerBetter = isLargerBetter;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if larger value is better for this metric.\n\t * \n\t * @return {@code true} if larger value is better for this metric.\n\t */\n\tpublic boolean isLargerBetter() {\n\t\treturn isLargerBetter;\n\t}\n\n\t/**\n\t * Returns {@code true} if the first value is better.\n\t * \n\t * @param a the 1st value.\n\t * @param b the 2nd value.\n\t * @return {@code true} if the first value is better.\n\t */\n\tpublic boolean isFirstBetter(double a, double b) {\n\t\treturn MathUtils.isFirstBetter(a, b, isLargerBetter);\n\t}\n\t\n\t/**\n\t * Returns the worst value of this metric.\n\t * \n\t * @return the worst value of this metric.\n\t */\n\tpublic double worstValue() {\n\t\tif (isLargerBetter) {\n\t\t\treturn Double.NEGATIVE_INFINITY;\n\t\t} else {\n\t\t\treturn Double.POSITIVE_INFINITY;\n\t\t}\n\t}\n\t\n\t/**\n\t * Evaluates predictions on a dataset.\n\t * \n\t * @param preds the predictions.\n\t * @param instances the dataset.\n\t * @return the evaluation measure.\n\t */\n\tpublic abstract double eval(double[] preds, Instances instances);\n\t\n\t/**\n\t * Returns the index of best metric value in a list.\n\t * \n\t * @param list the list of metric values.\n\t * @return the index of best metric value in a list.\n\t */\n\tpublic int searchBestMetricValueIndex(List<Double> list) {\n\t\tdouble bestSoFar = worstValue();\n\t\tint idx = -1;\n\t\tfor (int i = 0; i < list.size(); i++) {\n\t\t\tif (isFirstBetter(list.get(i), bestSoFar)) {\n\t\t\t\tbestSoFar = list.get(i);\n\t\t\t\tidx = i;\n\t\t\t}\n\t\t}\n\t\treturn idx;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/MetricFactory.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Factory class for creating metrics.\n *  \n * @author Yin Lou\n *\n */\npublic class MetricFactory {\n\t\n\tprivate static Map<String, Metric> map;\n\t\n\tstatic {\n\t\tmap = new HashMap<>();\n\t\tmap.put(\"auc\", new AUC());\n\t\tmap.put(\"error\", new Error());\n\t\tmap.put(\"logisticloss\", new LogisticLoss());\n\t\tmap.put(\"logloss\", new LogLoss(false));\n\t\tmap.put(\"logloss_t\", new LogLoss(true));\n\t\tmap.put(\"mae\", new MAE());\n\t\tmap.put(\"rmse\", new RMSE());\n\t}\n\n\t/**\n\t * Returns the metric.\n\t * \n\t * @param str the metric string.\n\t * @return the metric.\n\t */\n\tpublic static Metric getMetric(String str) {\n\t\tString[] data = str.toLowerCase().split(\":\");\n\t\tString name = data[0];\n\t\tif (data.length == 1) {\n\t\t\tif (!map.containsKey(name)) {\n\t\t\t\tthrow new IllegalArgumentException(\"Unrecognized metric name: \" + name);\n\t\t\t} else {\n\t\t\t\treturn map.get(name);\n\t\t\t}\n\t\t} else {\n\t\t\tif (name.equals(\"logloss\")) {\n\t\t\t\tif (data[1].startsWith(\"t\")) {\n\t\t\t\t\treturn map.get(\"logloss_t\");\n\t\t\t\t} else {\n\t\t\t\t\treturn map.get(name);\n\t\t\t\t}\n\t\t\t} else if (map.containsKey(name)) {\n\t\t\t\treturn map.get(name);\n\t\t\t}\n\t\t}\n\t\treturn null;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/Predictor.java",
    "content": "package mltk.predictor.evaluation;\n\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Classifier;\nimport mltk.predictor.Learner.Task;\nimport mltk.predictor.ProbabilisticClassifier;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.OptimUtils;\n\n/**\n * Class for making predictions.\n * \n * @author Yin Lou\n * \n */\npublic class Predictor {\n\t\n\tstatic class Options {\n\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\n\t\tString attPath = null;\n\n\t\t@Argument(name = \"-d\", description = \"data set path\", required = true)\n\t\tString dataPath = null;\n\n\t\t@Argument(name = \"-m\", description = \"model path\", required = true)\n\t\tString modelPath = null;\n\n\t\t@Argument(name = \"-p\", description = \"prediction path\")\n\t\tString predictionPath = null;\n\n\t\t@Argument(name = \"-R\", description = \"residual path\")\n\t\tString residualPath = null;\n\n\t\t@Argument(name = \"-g\", description = \"task between classification (c) and regression (r) (default: r)\")\n\t\tString task = \"r\";\n\n\t\t@Argument(name = \"-P\", description = \"output probablity (default: false)\")\n\t\tboolean prob = false;\n\n\t}\n\n\t/**\n\t * Makes predictions on a dataset.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.evaluation.Predictor\n\t * -d\tdata set path\n\t * -m\tmodel path\n\t * [-r]\tattribute file path\n\t * [-p]\tprediction path\n\t * [-R]\tresidual path\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-P]\toutput probability (default: false)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(Predictor.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tInstances instances = InstancesReader.read(opts.attPath, opts.dataPath);\n\t\tmltk.predictor.Predictor predictor = PredictorReader.read(opts.modelPath);\n\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tRegressor regressor = (Regressor) predictor;\n\t\t\t\tdouble rmse = Evaluator.evalRMSE(regressor, instances);\n\t\t\t\tSystem.out.println(\"RMSE on Test: \" + rmse);\n\n\t\t\t\tif (opts.predictionPath != null) {\n\t\t\t\t\tPrintWriter out = new PrintWriter(opts.predictionPath);\n\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\tdouble pred = regressor.regress(instance);\n\t\t\t\t\t\tout.println(pred);\n\t\t\t\t\t}\n\t\t\t\t\tout.flush();\n\t\t\t\t\tout.close();\n\t\t\t\t}\n\n\t\t\t\tif (opts.residualPath != null) {\n\t\t\t\t\tPrintWriter out = new PrintWriter(opts.residualPath);\n\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\tdouble pred = regressor.regress(instance);\n\t\t\t\t\t\tout.println(instance.getTarget() - pred);\n\t\t\t\t\t}\n\t\t\t\t\tout.flush();\n\t\t\t\t\tout.close();\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tClassifier classifier = (Classifier) predictor;\n\t\t\t\tdouble error = Evaluator.evalError(classifier, instances);\n\t\t\t\tSystem.out.println(\"Error rate on Test: \" + (error * 100) + \" %\");\n\n\t\t\t\tif (opts.predictionPath != null) {\n\t\t\t\t\tif (opts.prob) {\n\t\t\t\t\t\tPrintWriter out = new PrintWriter(opts.predictionPath);\n\t\t\t\t\t\tProbabilisticClassifier probClassifier = (ProbabilisticClassifier) predictor;\n\t\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\t\tdouble[] pred = probClassifier.predictProbabilities(instance);\n\t\t\t\t\t\t\tout.println(Arrays.toString(pred));\n\t\t\t\t\t\t}\n\t\t\t\t\t\tout.flush();\n\t\t\t\t\t\tout.close();\n\t\t\t\t\t} else {\n\t\t\t\t\t\tPrintWriter out = new PrintWriter(opts.predictionPath);\n\t\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\t\tdouble pred = classifier.classify(instance);\n\t\t\t\t\t\t\tout.println((int) pred);\n\t\t\t\t\t\t}\n\t\t\t\t\t\tout.flush();\n\t\t\t\t\t\tout.close();\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (opts.residualPath != null) {\n\t\t\t\t\tif (predictor instanceof Regressor) {\n\t\t\t\t\t\tPrintWriter out = new PrintWriter(opts.residualPath);\n\t\t\t\t\t\tRegressor regressingClassifier = (Regressor) predictor;\n\t\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\t\tdouble pred = regressingClassifier.regress(instance);\n\t\t\t\t\t\t\tint cls = (int) instance.getTarget();\n\t\t\t\t\t\t\tout.println(OptimUtils.getPseudoResidual(pred, cls));\n\t\t\t\t\t\t}\n\t\t\t\t\t\tout.flush();\n\t\t\t\t\t\tout.close();\n\t\t\t\t\t} else {\n\t\t\t\t\t\tSystem.out.println(\"Warning: Classifier does not support outputing pseudo residual.\");\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n\t/**\n\t * Makes predictions for a dataset.\n\t * \n\t * @param regressor the model.\n\t * @param instances the dataset.\n\t * @param path the output path.\n\t * @param residual {@code true} if residuals are the output.\n\t * @throws IOException\n\t */\n\tpublic static void predict(Regressor regressor, Instances instances, String path, boolean residual)\n\t\t\tthrows IOException {\n\t\tPrintWriter out = new PrintWriter(path);\n\t\tif (residual) {\n\t\t\tfor (Instance instance : instances) {\n\t\t\t\tdouble pred = regressor.regress(instance);\n\t\t\t\tout.println(instance.getTarget() - pred);\n\t\t\t}\n\t\t} else {\n\t\t\tfor (Instance instance : instances) {\n\t\t\t\tdouble pred = regressor.regress(instance);\n\t\t\t\tout.println(pred);\n\t\t\t}\n\t\t}\n\t\tout.flush();\n\t\tout.close();\n\t}\n\n\t/**\n\t * Makes predictions for a dataset.\n\t * \n\t * @param classifier the model.\n\t * @param instances the dataset.\n\t * @param path the output path.\n\t * @throws IOException\n\t */\n\tpublic static void predict(Classifier classifier, Instances instances, String path) throws IOException {\n\t\tPrintWriter out = new PrintWriter(path);\n\t\tfor (Instance instance : instances) {\n\t\t\tint pred = classifier.classify(instance);\n\t\t\tout.println(pred);\n\t\t}\n\t\tout.flush();\n\t\tout.close();\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/RMSE.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.core.Instances;\n\n/**\n * Class for evaluating root mean squared error (RMSE).\n * \n * @author Yin Lou\n *\n */\npublic class RMSE extends SimpleMetric {\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic RMSE() {\n\t\tsuper(false);\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, double[] targets) {\n\t\tdouble rmse = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tdouble d = targets[i] - preds[i];\n\t\t\trmse += d * d;\n\t\t}\n\t\trmse = Math.sqrt(rmse / preds.length);\n\t\treturn rmse;\n\t}\n\n\t@Override\n\tpublic double eval(double[] preds, Instances instances) {\n\t\tdouble rmse = 0;\n\t\tfor (int i = 0; i < preds.length; i++) {\n\t\t\tdouble d = instances.get(i).getTarget() - preds[i];\n\t\t\trmse += d * d;\n\t\t}\n\t\trmse = Math.sqrt(rmse / preds.length);\n\t\treturn rmse;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/SimpleMetric.java",
    "content": "package mltk.predictor.evaluation;\n\n/**\n * Class for simple metrics.\n * \n * @author Yin Lou\n *\n */\npublic abstract class SimpleMetric extends Metric {\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param isLargerBetter {@code true} if larger value is better.\n\t */\n\tpublic SimpleMetric(boolean isLargerBetter) {\n\t\tsuper(isLargerBetter);\n\t}\n\t\n\t/**\n\t * Evaluates predictions given targets.\n\t * \n\t * @param preds the predictions.\n\t * @param targets the targets.\n\t * @return the evaluation measure.\n\t */\n\tpublic abstract double eval(double[] preds, double[] targets);\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/evaluation/package-info.java",
    "content": "/**\r\n * Provides classes for evaluating predictors.\r\n */\r\npackage mltk.predictor.evaluation;"
  },
  {
    "path": "src/main/java/mltk/predictor/function/Array1D.java",
    "content": "package mltk.predictor.function;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\n\n/**\n * Class for 1D lookup tables.\n * \n * @author Yin Lou\n * \n */\npublic class Array1D implements Regressor, UnivariateFunction {\n\n\t/**\n\t * Attribute index. Must be binned/nominal attribute; otherwise the behavior is not guaranteed.\n\t */\n\tprotected int attIndex;\n\n\t/**\n\t * Predictions.\n\t */\n\tprotected double[] predictions;\n\t\n\t/**\n\t * Prediction on missing value.\n\t */\n\tprotected double predictionOnMV;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Array1D() {\n\n\t}\n\t\n\t/**\n\t * Constructs a 1D lookup table.\n\t * \n\t * @param attIndex the attribute index. The attribute must be discretized or nominal.\n\t * @param predictions the prediction array.\n\t */\n\tpublic Array1D(int attIndex, double[] predictions) {\n\t\tthis(attIndex, predictions, 0.0);\n\t}\n\n\t/**\n\t * Constructs a 1D lookup table.\n\t * \n\t * @param attIndex the attribute index. The attribute must be discretized or nominal.\n\t * @param predictions the prediction array.\n\t * @param predictionOnMV the prediction on missing value.\n\t */\n\tpublic Array1D(int attIndex, double[] predictions, double predictionOnMV) {\n\t\tthis.attIndex = attIndex;\n\t\tthis.predictions = predictions;\n\t\tthis.predictionOnMV = predictionOnMV;\n\t}\n\n\t/**\n\t * Returns the attribute index.\n\t * \n\t * @return the attribute index.\n\t */\n\tpublic int getAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\n\t/**\n\t * Sets the attribute index.\n\t * \n\t * @param attIndex the new attribute index.\n\t */\n\tpublic void setAttributeIndex(int attIndex) {\n\t\tthis.attIndex = attIndex;\n\t}\n\n\t/**\n\t * Returns the internal prediction array.\n\t * \n\t * @return the internal prediction array.\n\t */\n\tpublic double[] getPredictions() {\n\t\treturn predictions;\n\t}\n\n\t/**\n\t * Sets the internal prediction array.\n\t * \n\t * @param predictions the new prediction array.\n\t */\n\tpublic void setPredictions(double[] predictions) {\n\t\tthis.predictions = predictions;\n\t}\n\t\n\t/**\n\t * Returns the prediction on missing value.\n\t * \n\t * @return the prediction on missing value.\n\t */\n\tpublic double getPredictionOnMV() {\n\t\treturn predictionOnMV;\n\t}\n\t\n\t/**\n\t * Sets prediction on missing value.\n\t * \n\t * @param predictionOnMV the prediction on missing value.\n\t */\n\tpublic void setPredictionOnMV(double predictionOnMV) {\n\t\tthis.predictionOnMV = predictionOnMV;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tString line = in.readLine();\n\t\tString[] data = line.split(\": \");\n\t\tattIndex = Integer.parseInt(data[1]);\n\t\t\n\t\tline = in.readLine();\n\t\tdata = line.split(\": \");\n\t\tpredictionOnMV = Double.parseDouble(data[1]);\n\n\t\tin.readLine();\n\t\tpredictions = ArrayUtils.parseDoubleArray(in.readLine());\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"AttIndex: \" + attIndex);\n\t\tout.println(\"PredictionOnMV: \" + predictionOnMV);\n\t\tout.println(\"Predictions: \" + predictions.length);\n\t\tout.println(Arrays.toString(predictions));\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\tdouble v = instance.getValue(attIndex);\n\t\tif (!Double.isNaN(v)) {\n\t\t\treturn predictions[(int) v];\n\t\t} else {\n\t\t\treturn predictionOnMV;\n\t\t}\n\t}\n\n\t/**\n\t * Adds this lookup table with another one.\n\t * \n\t * @param ary the other lookup table.\n\t * @return this lookup table.\n\t */\n\tpublic Array1D add(Array1D ary) {\n\t\tif (attIndex != ary.attIndex) {\n\t\t\tthrow new IllegalArgumentException(\"Cannot add arrays on different terms\");\n\t\t}\n\t\tfor (int i = 0; i < predictions.length; i++) {\n\t\t\tpredictions[i] += ary.predictions[i];\n\t\t}\n\t\tpredictionOnMV += ary.predictionOnMV;\n\t\treturn this;\n\t}\n\n\t@Override\n\tpublic double evaluate(double x) {\n\t\tif (!Double.isNaN(x)) {\n\t\t\treturn predictions[(int) x];\n\t\t} else {\n\t\t\treturn predictionOnMV;\n\t\t}\n\t}\n\n\t@Override\n\tpublic Array1D copy() {\n\t\tdouble[] predictionsCopy = Arrays.copyOf(predictions, predictions.length);\n\t\treturn new Array1D(attIndex, predictionsCopy, predictionOnMV);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/Array2D.java",
    "content": "package mltk.predictor.function;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\nimport mltk.util.tuple.IntPair;\n\n/**\n * Class for 2D lookup tables.\n * \n * @author Yin Lou\n * \n */\npublic class Array2D implements Regressor, BivariateFunction {\n\n\t/**\n\t * First attribute index.\n\t */\n\tprotected int attIndex1;\n\n\t/**\n\t * Second attribute index.\n\t */\n\tprotected int attIndex2;\n\n\t/**\n\t * Predictions.\n\t */\n\tprotected double[][] predictions;\n\t\n\t/**\n\t * Predictions on missing value for attribute 1.\n\t */\n\tprotected double[] predictionsOnMV1;\n\t\n\t/**\n\t * Predictions on missing value for attribute 2.\n\t */\n\tprotected double[] predictionsOnMV2;\n\t\n\t/**\n\t * Prediction when both attribute 1 and 2 are missing.\n\t */\n\tprotected double predictionOnMV12;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Array2D() {\n\n\t}\n\n\t/**\n\t * Constructs a 2D lookup table.\n\t * \n\t * @param attIndex1 the 1st attribute index. The attribute must be discretized or nominal.\n\t * @param attIndex2 the 2nd attribute index. The attribute must be discretized or nominal.\n\t * @param predictions the prediction matrix.\n\t */\n\tpublic Array2D(int attIndex1, int attIndex2, double[][] predictions) {\n\t\tthis.attIndex1 = attIndex1;\n\t\tthis.attIndex2 = attIndex2;\n\t\tthis.predictions = predictions;\n\t}\n\t\n\t/**\n\t * Constructs a 2D lookup table.\n\t * \n\t * @param attIndex1 the 1st attribute index. The attribute must be discretized or nominal.\n\t * @param attIndex2 the 2nd attribute index. The attribute must be discretized or nominal.\n\t * @param predictions the prediction matrix.\n\t * @param predictionsOnMV1 the prediction array when the 1st attribute is missing.\n\t * @param predictionsOnMV2 the prediction array when the 2nd attribute is missing.\n\t * @param predictionOnMV12 the prediction when both attributes are missing.\n\t */\n\tpublic Array2D(int attIndex1, int attIndex2, double[][] predictions,\n\t\t\tdouble[] predictionsOnMV1, double[] predictionsOnMV2, double predictionOnMV12) {\n\t\tthis.attIndex1 = attIndex1;\n\t\tthis.attIndex2 = attIndex2;\n\t\tthis.predictions = predictions;\n\t\tthis.predictionsOnMV1 = predictionsOnMV1;\n\t\tthis.predictionsOnMV2 = predictionsOnMV2;\n\t\tthis.predictionOnMV12 = predictionOnMV12;\n\t}\n\n\t/**\n\t * Returns the index of 1st attribute.\n\t * \n\t * @return the index of 1st attribute.\n\t */\n\tpublic int getAttributeIndex1() {\n\t\treturn attIndex1;\n\t}\n\n\t/**\n\t * Returns the index of 2nd attribute.\n\t * \n\t * @return the index of 2nd attribute.\n\t */\n\tpublic int getAttributeIndex2() {\n\t\treturn attIndex2;\n\t}\n\n\t/**\n\t * Returns the attribute indices pair.\n\t * \n\t * @return the attribute indices pair.\n\t */\n\tpublic IntPair getAttributeIndices() {\n\t\treturn new IntPair(attIndex1, attIndex2);\n\t}\n\n\t/**\n\t * Sets the attribute indices.\n\t * \n\t * @param attIndex1 the new 1st attribute index.\n\t * @param attIndex2 the new 2nd attribute index.\n\t */\n\tpublic void setAttributeIndices(int attIndex1, int attIndex2) {\n\t\tthis.attIndex1 = attIndex1;\n\t\tthis.attIndex2 = attIndex2;\n\t}\n\n\t/**\n\t * Returns the internal prediction matrix.\n\t * \n\t * @return the internal prediction matrix.\n\t */\n\tpublic double[][] getPredictions() {\n\t\treturn predictions;\n\t}\n\n\t/**\n\t * Sets the internal prediction matrix.\n\t * \n\t * @param predictions the new prediction matrix.\n\t */\n\tpublic void setPredictions(double[][] predictions) {\n\t\tthis.predictions = predictions;\n\t}\n\t\n\t/**\n\t * Returns the internal prediction array when the 1st attribute is missing.\n\t * \n\t * @return the internal prediction array when the 1st attribute is missing.\n\t */\n\tpublic double[] getPredictionsOnMV1() {\n\t\treturn predictionsOnMV1;\n\t}\n\t\n\t/**\n\t * Sets the internal prediction array when the 1st attribute is missing.\n\t * \n\t * @param predictionsOnMV1 the new prediction array.\n\t */\n\tpublic void setPredictionsOnMV1(double[] predictionsOnMV1) {\n\t\tthis.predictionsOnMV1 = predictionsOnMV1;\n\t}\n\t\n\t/**\n\t * Returns the internal prediction array when the 2nd attribute is missing.\n\t * \n\t * @return the internal prediction array when the 2nd attribute is missing.\n\t */\n\tpublic double[] getPredictionsOnMV2() {\n\t\treturn predictionsOnMV2;\n\t}\n\t\n\t/**\n\t * Sets the internal prediction array when the 2nd attribute is missing.\n\t * \n\t * @param predictionsOnMV2 the new prediction array.\n\t */\n\tpublic void setPredictionsOnMV2(double[] predictionsOnMV2) {\n\t\tthis.predictionsOnMV2 = predictionsOnMV2;\n\t}\n\t\n\t/**\n\t * Returns the prediction when both attributes are missing.\n\t * \n\t * @return the prediction when both attributes are missing.\n\t */\n\tpublic double getPredictionOnMV12() {\n\t\treturn predictionOnMV12;\n\t}\n\t\n\t/**\n\t * Sets the prediction when both attributes are missing.\n\t * \n\t * @param predictionOnMV12 the new prediction.\n\t */\n\tpublic void setPredictionOnMV12(double predictionOnMV12) {\n\t\tthis.predictionOnMV12 = predictionOnMV12;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tString line = in.readLine();\n\t\tString[] data = line.split(\": \");\n\t\tattIndex1 = Integer.parseInt(data[1]);\n\t\tline = in.readLine();\n\t\tdata = line.split(\": \");\n\t\tattIndex2 = Integer.parseInt(data[1]);\n\n\t\tString[] dim = in.readLine().split(\": \")[1].split(\"x\");\n\t\tpredictions = new double[Integer.parseInt(dim[0])][];\n\t\tfor (int i = 0; i < predictions.length; i++) {\n\t\t\tpredictions[i] = ArrayUtils.parseDoubleArray(in.readLine());\n\t\t}\n\t\t\n\t\tin.readLine();\n\t\tpredictionsOnMV1 = ArrayUtils.parseDoubleArray(in.readLine());\n\t\t\n\t\tin.readLine();\n\t\tpredictionsOnMV2 = ArrayUtils.parseDoubleArray(in.readLine());\n\t\t\n\t\tdata = in.readLine().split(\": \");\n\t\tpredictionOnMV12 = Double.parseDouble(data[1]);\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"AttIndex1: \" + attIndex1);\n\t\tout.println(\"AttIndex2: \" + attIndex2);\n\t\tout.println(\"Predictions: \" + predictions.length + \"x\" + predictions[0].length);\n\t\tfor (int i = 0; i < predictions.length; i++) {\n\t\t\tout.println(Arrays.toString(predictions[i]));\n\t\t}\n\t\tout.println(\"PredictionsOnMV1: \" + predictionsOnMV1.length);\n\t\tout.println(Arrays.toString(predictionsOnMV1));\n\t\tout.println(\"PredictionsOnMV2: \" + predictionsOnMV2.length);\n\t\tout.println(Arrays.toString(predictionsOnMV2));\n\t\tout.println(\"PredictionOnMV12: \" + predictionOnMV12);\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\treturn evaluate(instance.getValue(attIndex1), instance.getValue(attIndex2));\n\t}\n\n\t/**\n\t * Adds this lookup table with another one.\n\t * \n\t * @param ary the other lookup table.\n\t * @return this lookup table.\n\t */\n\tpublic Array2D add(Array2D ary) {\n\t\tif (attIndex1 != ary.attIndex1 || attIndex2 != ary.attIndex2) {\n\t\t\tthrow new IllegalArgumentException(\"Cannot add arrays on differnt terms\");\n\t\t}\n\t\tfor (int i = 0; i < predictions.length; i++) {\n\t\t\tpredictionsOnMV2[i] += ary.predictionsOnMV2[i];\n\t\t\t\n\t\t\tdouble[] preds1 = predictions[i];\n\t\t\tdouble[] preds2 = ary.predictions[i];\n\t\t\tfor (int j = 0; j < preds1.length; j++) {\n\t\t\t\tpredictionsOnMV1[j] += ary.predictionsOnMV1[j];\n\t\t\t\t\n\t\t\t\tpreds1[j] += preds2[j];\n\t\t\t}\n\t\t}\n\t\tpredictionOnMV12 += ary.predictionOnMV12;\n\t\treturn this;\n\t}\n\n\t@Override\n\tpublic double evaluate(double x, double y) {\n\t\tif (!Double.isNaN(x) && !Double.isNaN(y)) {\n\t\t\treturn predictions[(int) x][(int) y];\n\t\t} else if (Double.isNaN(x) && !Double.isNaN(y)) {\n\t\t\treturn predictionsOnMV1[(int) y];\n\t\t} else if (!Double.isNaN(x) && Double.isNaN(y)) {\n\t\t\treturn predictionsOnMV2[(int) x];\n\t\t} else {\n\t\t\treturn predictionOnMV12;\n\t\t}\n\t\t\n\t}\n\n\t@Override\n\tpublic Array2D copy() {\n\t\tdouble[][] predictionsCopy = new double[predictions.length][];\n\t\tfor (int i = 0; i < predictionsCopy.length; i++) {\n\t\t\tpredictionsCopy[i] = Arrays.copyOf(predictions[i], predictions[i].length);\n\t\t}\n\t\tdouble[] predictionsOnMV1Copy = Arrays.copyOf(predictionsOnMV1, predictionsOnMV1.length);\n\t\tdouble[] predictionsOnMV2Copy = Arrays.copyOf(predictionsOnMV2, predictionsOnMV2.length);\n\t\treturn new Array2D(attIndex1, attIndex2, predictionsCopy,\n\t\t\t\tpredictionsOnMV1Copy, predictionsOnMV2Copy, predictionOnMV12);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/BaggedLineCutter.java",
    "content": "package mltk.predictor.function;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.Sampling;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.util.tuple.IntPair;\nimport mltk.util.Element;\nimport mltk.util.MathUtils;\n\n/**\n * Class for cutting lines with bagging.\n * \n * @author Yin Lou\n *\n */\npublic class BaggedLineCutter extends EnsembledLineCutter {\n\t\n\tprivate List<IntPair[]> samples;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic BaggedLineCutter() {\n\t\tthis(false);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param isClassification {@code true} if it is a classification problem.\n\t */\n\tpublic BaggedLineCutter(boolean isClassification) {\n\t\tattIndex = -1;\n\t\tthis.isClassification = isClassification;\n\t}\n\t\n\t/**\n\t * Creates internal bootstrap samples.\n\t * \n\t * @param n the size of the dataset to sample.\n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void createBags(int n, int baggingIters) {\n\t\tsamples = new ArrayList<>(baggingIters);\n\t\tif (baggingIters <= 0) {\n\t\t\t// No bagging\n\t\t\tIntPair[] indices = new IntPair[n];\n\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\tindices[i] = new IntPair(i, 1);\n\t\t\t}\n\t\t\tsamples.add(indices);\n\t\t} else {\n\t\t\tfor (int b = 0; b < baggingIters; b++) {\n\t\t\t\tsamples.add(Sampling.createBootstrapSampleIndices(n));\n\t\t\t}\n\t\t}\n\t}\n\n\t@Override\n\tpublic BaggedEnsemble build(Instances instances) {\n\t\treturn build(instances, attIndex, numIntervals);\n\t}\n\n\t@Override\n\tpublic BaggedEnsemble build(Instances instances, Attribute attribute, int numIntervals) {\n\t\tint attIndex = attribute.getIndex();\n\t\t\n\t\tif (samples == null) {\n\t\t\tcreateBags(instances.size(), baggingIters);\n\t\t}\n\t\t\n\t\tBaggedEnsemble ensemble = new BaggedEnsemble(samples.size());\n\t\t\n\t\tdouble[] targets = new double[instances.size()];\n\t\tdouble[] fvalues = new double[instances.size()];\n\t\tdouble[] weights = new double[instances.size()];\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\ttargets[i] = instance.getTarget();\n\t\t\tfvalues[i] = instance.getValue(attribute);\n\t\t\tweights[i] = instance.getWeight();\n\t\t}\n\t\t\n\t\tif (attribute.getType() == Attribute.Type.NUMERIC) {\n\t\t\tfor (IntPair[] indices : samples) {\n\t\t\t\tdouble sumRespOnMV = 0.0;\n\t\t\t\tdouble sumWeightOnMV = 0.0;\n\t\t\t\t\n\t\t\t\tList<Element<double[]>> pairs = new ArrayList<>(instances.size());\n\t\t\t\tfor (IntPair entry : indices) {\n\t\t\t\t\tint index = entry.v1;\n\t\t\t\t\tint w = entry.v2;\n\t\t\t\t\tdouble weight = weights[index];\n\t\t\t\t\tdouble value = fvalues[index];\n\t\t\t\t\tdouble target = targets[index];\n\t\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\t\tpairs.add(new Element<double[]>(new double[] { target * w, weight * w }, value));\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tpairs.add(new Element<double[]>(new double[] { target * weight * w, weight * w }, value));\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\t\tsumRespOnMV += target * w;\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tsumRespOnMV += target * weight * w;\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsumWeightOnMV += weight * w;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tCollections.sort(pairs);\n\t\t\t\tList<double[]> histograms = new ArrayList<>();\n\t\t\t\tLineCutter.getHistograms(pairs, histograms);\n\t\t\t\thistograms.add(new double[] { Double.NaN, sumRespOnMV, sumWeightOnMV });\n\t\t\t\t\n\t\t\t\tFunction1D func = LineCutter.build(attIndex, histograms, numIntervals);\n\t\t\t\tensemble.add(func);\n\t\t\t}\n\t\t} else {\n\t\t\tint size = 0;\n\t\t\tif (attribute.getType() == Attribute.Type.BINNED) {\n\t\t\t\tsize = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t} else {\n\t\t\t\tsize = ((NominalAttribute) attribute).getCardinality();\n\t\t\t}\n\t\t\tfor (IntPair[] indices : samples) {\n\t\t\t\tList<double[]> histograms;\n\t\t\t\tdouble sumRespOnMV = 0.0;\n\t\t\t\tdouble sumWeightOnMV = 0.0;\n\t\t\t\t\n\t\t\t\tdouble[][] histogram = new double[size][3];\n\t\t\t\tfor (IntPair entry : indices) {\n\t\t\t\t\tint index = entry.v1;\n\t\t\t\t\tint w = entry.v2;\n\t\t\t\t\tdouble weight = weights[index];\n\t\t\t\t\tdouble value = fvalues[index];\n\t\t\t\t\tdouble target = targets[index];\n\t\t\t\t\t\n\t\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\t\tint idx = (int) value;\n\t\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\t\thistogram[idx][0] += target * w;\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\thistogram[idx][0] += target * weight * w;\n\t\t\t\t\t\t}\n\t\t\t\t\t\thistogram[idx][1] += weight * w;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\t\tsumRespOnMV += target * w;\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tsumRespOnMV += target * weight * w;\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsumWeightOnMV += weight * w;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\thistograms = new ArrayList<>(histogram.length + 1);\n\t\t\t\tfor (int i = 0; i < histogram.length; i++) {\n\t\t\t\t\tif (!MathUtils.isZero(histogram[i][1])) {\n\t\t\t\t\t\tdouble[] hist = histogram[i];\n\t\t\t\t\t\thistograms.add(new double[] {i, hist[0], hist[1]});\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\thistograms.add(new double[] { Double.NaN, sumRespOnMV, sumWeightOnMV });\n\t\t\t\t\n\t\t\t\tFunction1D func = LineCutter.build(attIndex, histograms, numIntervals);\n\t\t\t\tensemble.add(func);\n\t\t\t}\n\t\t}\n\t\t\n\t\treturn ensemble;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/BivariateFunction.java",
    "content": "package mltk.predictor.function;\n\n/**\n * Interface for bivariate real functions.\n * \n * @author Yin Lou\n * \n */\npublic interface BivariateFunction {\n\n\t/**\n\t * Computes the value for the function.\n\t * \n\t * @param x the 1st argument.\n\t * @param y the 2nd argument.\n\t * @return the value for the function.\n\t */\n\tpublic double evaluate(double x, double y);\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/CHistogram.java",
    "content": "package mltk.predictor.function;\r\n\r\n/**\r\n * Class for cumulative histograms.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class CHistogram {\r\n\r\n\tpublic double[] sum;\r\n\tpublic double[] count;\r\n\tpublic double sumOnMV;\r\n\tpublic double countOnMV;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param n the size of this cumulative histogram.\r\n\t */\r\n\tpublic CHistogram(int n) {\r\n\t\tsum = new double[n];\r\n\t\tcount = new double[n];\r\n\t\tsumOnMV = 0.0;\r\n\t\tcountOnMV = 0.0;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the size of this cumulative histogram.\r\n\t * \r\n\t * @return the size of this cumulative histogram.\r\n\t */\r\n\tpublic int size() {\r\n\t\treturn sum.length;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns {@code true} if missing values are present.\r\n\t * \r\n\t * @return {@code true} if missing values are present.\r\n\t */\r\n\tpublic boolean hasMissingValue() {\r\n\t\treturn countOnMV > 0;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/CompressionUtils.java",
    "content": "package mltk.predictor.function;\n\nimport mltk.core.Instance;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.predictor.BoostedEnsemble;\nimport mltk.predictor.Predictor;\nimport mltk.util.tuple.IntPair;\n\n/**\n * Class for utility functions for compressing ensembles of univariate/bivariate functions.\n * \n * @author Yin Lou\n * \n */\npublic class CompressionUtils {\n\n\t/**\n\t * Compresses a bagged ensemble of 1D functions to a single 1D function.\n\t * \n\t * @param attIndex the attribute index of this regressor.\n\t * @param baggedEnsemble the bagged ensemble.\n\t * @return a single compressed 1D function.\n\t */\n\tpublic static Function1D compress(int attIndex, BaggedEnsemble baggedEnsemble) {\n\t\tFunction1D function = Function1D.getConstantFunction(attIndex, 0);\n\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\tPredictor predictor = baggedEnsemble.get(i);\n\t\t\tFunction1D func = null;\n\t\t\tif (predictor instanceof Function1D) {\n\t\t\t\tfunc = (Function1D) predictor;\n\t\t\t} else {\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t\tfunction.add(func);\n\t\t}\n\t\tfunction.divide(baggedEnsemble.size());\n\t\treturn function;\n\t}\n\n\t/**\n\t * Compresses a boosted ensemble to a single 1D function.\n\t * \n\t * @param attIndex the attribute of this regressor.\n\t * @param boostedEnsemble the boosted ensemble.\n\t * @return a single compressed 1D function.\n\t */\n\tpublic static Function1D compress(int attIndex, BoostedEnsemble boostedEnsemble) {\n\t\tFunction1D function = Function1D.getConstantFunction(attIndex, 0);\n\t\tfor (int i = 0; i < boostedEnsemble.size(); i++) {\n\t\t\tPredictor predictor = boostedEnsemble.get(i);\n\t\t\tFunction1D func = null;\n\t\t\tif (predictor instanceof Function1D) {\n\t\t\t\tfunc = (Function1D) predictor;\n\t\t\t} else if (predictor instanceof BaggedEnsemble) {\n\t\t\t\tfunc = compress(attIndex, (BaggedEnsemble) predictor);\n\t\t\t} else {\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t\tfunction.add(func);\n\t\t}\n\t\treturn function;\n\t}\n\n\t/**\n\t * Converts a 1D function to 1D lookup table.\n\t * \n\t * @param n the number of bins.\n\t * @param function the 1D function.\n\t * @return a 1D lookup table.\n\t */\n\tpublic static Array1D convert(int n, Function1D function) {\n\t\tdouble[] predictions = new double[n];\n\t\tfor (int i = 0; i < predictions.length; i++) {\n\t\t\tpredictions[i] = function.evaluate(i);\n\t\t}\n\t\treturn new Array1D(function.getAttributeIndex(), predictions, function.predictionOnMV);\n\t}\n\n\t/**\n\t * Compresses a bagged ensemble of 2D functions to a single 2D function.\n\t * \n\t * @param attIndex1 the 1st attribute index.\n\t * @param attIndex2 the 2nd attribute index.\n\t * @param baggedEnsemble the bagged ensemble.\n\t * @return a single compressed 2D functions.\n\t */\n\tpublic static Function2D compress(int attIndex1, int attIndex2, BaggedEnsemble baggedEnsemble) {\n\t\t// TODO check consistency problem when missing values are present.\n\t\tFunction2D function = Function2D.getConstantFunction(attIndex1, attIndex2, 0);\n\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\tPredictor predictor = baggedEnsemble.get(i);\n\t\t\tFunction2D func = null;\n\t\t\tif (predictor instanceof Function2D) {\n\t\t\t\tfunc = (Function2D) predictor;\n\t\t\t} else {\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t\tfunction.add(func);\n\t\t}\n\t\tfunction.divide(baggedEnsemble.size());\n\t\treturn function;\n\t}\n\n\t/**\n\t * Compresses a boosted ensemble to a single 2D function.\n\t * \n\t * @param attIndex1 the 1st attribute index.\n\t * @param attIndex2 the 2nd attribute index.\n\t * @param boostedEnsemble the boosted ensemble.\n\t * @return a single compressed 2D function.\n\t */\n\tpublic static Function2D compress(int attIndex1, int attIndex2, BoostedEnsemble boostedEnsemble) {\n\t\t// TODO check consistency problem when missing values are present.\n\t\tFunction2D function = Function2D.getConstantFunction(attIndex1, attIndex2, 0);\n\t\tfor (int i = 0; i < boostedEnsemble.size(); i++) {\n\t\t\tPredictor predictor = boostedEnsemble.get(i);\n\t\t\tFunction2D func = null;\n\t\t\tif (predictor instanceof Function2D) {\n\t\t\t\tfunc = (Function2D) predictor;\n\t\t\t} else if (predictor instanceof BaggedEnsemble) {\n\t\t\t\tfunc = compress(attIndex1, attIndex2, (BaggedEnsemble) predictor);\n\t\t\t} else {\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t\tfunction.add(func);\n\t\t}\n\t\treturn function;\n\t}\n\t\n\t/**\n\t * Compresses and converts a boosted ensemble to a single 2D lookup table.\n\t * \n\t * @param attIndex1 the 1st attribute index.\n\t * @param attIndex2 the 2nd attribute index.\n\t * @param boostedEnsemble the boosted ensemble.\n\t * @return a 2D lookup table.\n\t */\n\tpublic static Array2D compress(int attIndex1, int attIndex2, int n1, int n2, BoostedEnsemble boostedEnsemble) {\n\t\tdouble[][] predictions = new double[n1][n2];\n\t\tdouble[] predictionsOnMV1 = new double[n2];\n\t\tdouble[] predictionsOnMV2 = new double[n1];\n\t\tdouble[] vector = new double[Math.max(attIndex1, attIndex2) + 1];\n\t\tInstance instance = new Instance(vector);\n\t\tfor (int i = 0; i < n1; i++) {\n\t\t\tvector[attIndex1] = i;\n\t\t\tvector[attIndex2] = Double.NaN;\n\t\t\tpredictionsOnMV2[i] = boostedEnsemble.regress(instance);\n\t\t\t\n\t\t\tdouble[] preds = predictions[i];\n\t\t\tfor (int j = 0; j < n2; j++) {\n\t\t\t\tvector[attIndex1] = Double.NaN;\n\t\t\t\tvector[attIndex2] = j;\n\t\t\t\tpredictionsOnMV1[j] = boostedEnsemble.regress(instance);\n\t\t\t\t\n\t\t\t\tvector[attIndex1] = i;\n\t\t\t\tpreds[j] = boostedEnsemble.regress(instance);\n\t\t\t}\n\t\t}\n\t\tvector[attIndex1] = Double.NaN;\n\t\tvector[attIndex2] = Double.NaN;\n\t\treturn new Array2D(attIndex1, attIndex2, predictions,\n\t\t\t\tpredictionsOnMV1, predictionsOnMV2, boostedEnsemble.regress(instance));\n\t}\n\n\t/**\n\t * Converts a 2D function to 2D lookup table.\n\t * \n\t * @param n1 the number of bins for 1st attribute.\n\t * @param n2 the number of bins for 2nd attribute.\n\t * @param function the 2D function.\n\t * @return a 2D lookup table.\n\t */\n\tpublic static Array2D convert(int n1, int n2, Function2D function) {\n\t\tdouble[][] predictions = new double[n1][n2];\n\t\tdouble[] predictionsOnMV1 = new double[n2];\n\t\tdouble[] predictionsOnMV2 = new double[n1];\n\t\tfor (int i = 0; i < n1; i++) {\n\t\t\tpredictionsOnMV2[i] = function.evaluate(i, Double.NaN);\n\t\t\t\n\t\t\tdouble[] preds = predictions[i];\n\t\t\tfor (int j = 0; j < n2; j++) {\n\t\t\t\tpredictionsOnMV1[j] = function.evaluate(Double.NaN, j);\n\t\t\t\t\n\t\t\t\tpreds[j] = function.evaluate(i, j);\n\t\t\t}\n\t\t}\n\t\tIntPair attIndices = function.getAttributeIndices();\n\t\treturn new Array2D(attIndices.v1, attIndices.v2, predictions,\n\t\t\t\tpredictionsOnMV1, predictionsOnMV2, function.predictionOnMV12);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/CubicSpline.java",
    "content": "package mltk.predictor.function;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\n\n/**\n * Class for cubic splines. Given knots k, the cubic spline uses the following basis: 1, x, x^2, x^3, (x - k[i])_+^3.\n * \n * @author Yin Lou\n * \n */\npublic class CubicSpline implements Regressor, UnivariateFunction {\n\n\tprotected int attIndex;\n\tprotected double intercept;\n\tprotected double[] knots;\n\tprotected double[] w;\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param attIndex the attribute index.\n\t * @param intercept the intercept.\n\t * @param knots the knots.\n\t * @param w the coefficient vector.\n\t */\n\tpublic CubicSpline(int attIndex, double intercept, double[] knots, double[] w) {\n\t\tthis.attIndex = attIndex;\n\t\tthis.intercept = intercept;\n\t\tthis.knots = knots;\n\t\tthis.w = w;\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param intercept the intercept.\n\t * @param knots the knots.\n\t * @param w the coefficient vector.\n\t */\n\tpublic CubicSpline(double intercept, double[] knots, double[] w) {\n\t\tthis(-1, intercept, knots, w);\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param knots the knots.\n\t * @param w the coefficient vector.\n\t */\n\tpublic CubicSpline(double[] knots, double[] w) {\n\t\tthis(-1, 0, knots, w);\n\t}\n\n\t/**\n\t * Construct a cubic spline with specified knots and all coefficients set to zero.\n\t * \n\t * @param knots the knots.\n\t */\n\tpublic CubicSpline(double[] knots) {\n\t\tthis(-1, 0, knots, new double[knots.length + 3]);\n\t}\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic CubicSpline() {\n\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tattIndex = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tintercept = Double.parseDouble(in.readLine().split(\": \")[1]);\n\t\tin.readLine();\n\t\tknots = ArrayUtils.parseDoubleArray(in.readLine());\n\t\tin.readLine();\n\t\tw = ArrayUtils.parseDoubleArray(in.readLine());\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"AttIndex: \" + attIndex);\n\t\tout.println(\"Intercept: \" + intercept);\n\t\tout.println(\"Knots: \" + knots.length);\n\t\tout.println(Arrays.toString(knots));\n\t\tout.println(\"Coefficients: \" + w.length);\n\t\tout.println(Arrays.toString(w));\n\t}\n\n\t@Override\n\tpublic double evaluate(double x) {\n\t\tdouble pred = intercept + w[0] * x + w[1] * x * x + w[2] * x * x * x;\n\t\tfor (int i = 0; i < knots.length; i++) {\n\t\t\tpred += h(x, knots[i]) * w[i + 3];\n\t\t}\n\t\treturn pred;\n\t}\n\n\t/**\n\t * Calculate the basis.\n\t * \n\t * @param x a real.\n\t * @param k a knot.\n\t * @return h(x, z), a basis in cubic spline.\n\t */\n\tpublic static double h(double x, double k) {\n\t\tdouble t = x - k;\n\t\tif (t < 0) {\n\t\t\treturn 0;\n\t\t}\n\t\treturn t * t * t;\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\treturn evaluate(instance.getValue(attIndex));\n\t}\n\n\t/**\n\t * Returns the attribute index.\n\t * \n\t * @return the attribute index.\n\t */\n\tpublic int getAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\n\t/**\n\t * Sets the attribute index.\n\t * \n\t * @param attIndex the new attribute index.\n\t */\n\tpublic void setAttributeIndex(int attIndex) {\n\t\tthis.attIndex = attIndex;\n\t}\n\n\t/**\n\t * Returns the coefficient vector.\n\t * \n\t * @return the coefficient vector.\n\t */\n\tpublic double[] getCoefficients() {\n\t\treturn w;\n\t}\n\n\t/**\n\t * Returns the knot vector.\n\t * \n\t * @return the knot vector.\n\t */\n\tpublic double[] getKnots() {\n\t\treturn knots;\n\t}\n\n\t/**\n\t * Returns the intercept.\n\t * \n\t * @return the intercept.\n\t */\n\tpublic double getIntercept() {\n\t\treturn intercept;\n\t}\n\n\t@Override\n\tpublic CubicSpline copy() {\n\t\tdouble[] newW = w.clone();\n\t\tdouble[] newKnots = knots.clone();\n\t\treturn new CubicSpline(attIndex, intercept, newW, newKnots);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/EnsembledLineCutter.java",
    "content": "package mltk.predictor.function;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.predictor.Learner;\n\npublic abstract class EnsembledLineCutter extends Learner {\n\t\n\tprotected int attIndex;\n\n\tprotected int numIntervals;\n\t\n\tprotected int baggingIters;\n\t\n\tprotected boolean isClassification;\n\n\t@Override\n\tpublic BaggedEnsemble build(Instances instances) {\n\t\treturn build(instances, attIndex, numIntervals);\n\t}\n\t\n\t/**\n\t * Builds an 1D function ensemble.\n\t * \n\t * @param instances the training set.\n\t * @param attIndex the attribute index.\n\t * @param numIntervals the number of intervals.\n\t * @return an 1D function ensemble.\n\t */\n\tpublic BaggedEnsemble build(Instances instances, int attIndex, int numIntervals) {\n\t\tAttribute attribute = instances.getAttributes().get(attIndex);\n\t\treturn build(instances, attribute, numIntervals);\n\t}\n\t\n\tpublic abstract BaggedEnsemble build(Instances instances, Attribute attribute, int numIntervals);\n\t\n\t/**\n\t * Returns the index in the attribute list of the training set.\n\t * \n\t * @return the index in the attribute list of the training set.\n\t */\n\tpublic int getAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\t\n\t/**\n\t * Sets the index in the attribute list of the training set.\n\t * \n\t * @param attIndex the attribute index.\n\t */\n\tpublic void setAttributeIndex(int attIndex) {\n\t\tthis.attIndex = attIndex;\n\t}\n\t\n\t/**\n\t * Returns the number of bagging iterations.\n\t * \n\t * @return the number of bagging iterations.\n\t */\n\tpublic int getBaggingIters() {\n\t\treturn baggingIters;\n\t}\n\n\t/**\n\t * Sets the number of bagging iterations.\n\t * \n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void setBaggingIters(int baggingIters) {\n\t\tthis.baggingIters = baggingIters;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if it is a classification problem.\n\t * \n\t * @return {@code true} if it is a classification problem.\n\t */\n\tpublic boolean isClassification() {\n\t\treturn isClassification;\n\t}\n\t\n\t/**\n\t * Sets {@code true} if it is a classification problem.\n\t * \n\t * @param isClassification {@code true} if it is a classification problem.\n\t */\n\tpublic void setClassification(boolean isClassification) {\n\t\tthis.isClassification = isClassification;\n\t}\n\t\n\t/**\n\t * Returns the number of intervals.\n\t * \n\t * @return the number of intervals.\n\t */\n\tpublic int getNumIntervals() {\n\t\treturn numIntervals;\n\t}\n\n\t/**\n\t * Sets the number of intervals.\n\t * \n\t * @param numIntervals the number of intervals.\n\t */\n\tpublic void setNumIntervals(int numIntervals) {\n\t\tthis.numIntervals = numIntervals;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/Function1D.java",
    "content": "package mltk.predictor.function;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\nimport mltk.util.MathUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for 1D functions.\n * \n * <p>\n * This class represents a segmented 1D function. Segments are defined in split array. For example, [3, 5, +INF] defines\n * three segments: (-INF, 3], (3, 5], (5, +INF). The last value in the split array is always +INF. The prediction array\n * is the corresponding predictions for segments defined in splits.\n * </p>\n * \n * @author Yin Lou\n * \n */\npublic class Function1D implements Regressor, UnivariateFunction {\n\n\t/**\n\t * Attribute index.\n\t */\n\tprotected int attIndex;\n\n\t/**\n\t * Last value is always Double.POSITIVE_INFINITY. e.g. [3, 5, +INF] defines three segments: (-INF, 3], (3, 5], (5,\n\t * +INF)\n\t */\n\tprotected double[] splits;\n\n\t/**\n\t * Corresponding predictions for segments defined in splits.\n\t */\n\tprotected double[] predictions;\n\t\n\t/**\n\t * Prediction on missing value.\n\t */\n\tprotected double predictionOnMV;\n\n\t/**\n\t * Returns a constant 1D function.\n\t * \n\t * @param attIndex the attribute index of this function.\n\t * @param prediction the constant.\n\t * @return a constant 1D function.\n\t */\n\tpublic static Function1D getConstantFunction(int attIndex, double prediction) {\n\t\tFunction1D func = new Function1D(attIndex, new double[] { Double.POSITIVE_INFINITY },\n\t\t\t\tnew double[] { prediction });\n\t\treturn func;\n\t}\n\n\t/**\n\t * Resets this function to 0.\n\t */\n\tpublic void setZero() {\n\t\tsplits = new double[] { Double.POSITIVE_INFINITY };\n\t\tpredictions = new double[] { 0 };\n\t\tpredictionOnMV = 0.0;\n\t}\n\n\t/**\n\t * Returns {@code true} if the function is 0.\n\t * \n\t * @return {@code true} if the function is 0.\n\t */\n\tpublic boolean isZero() {\n\t\treturn ArrayUtils.isConstant(predictions, 0, predictions.length, 0)\n\t\t\t\t&& MathUtils.isZero(predictionOnMV);\n\t}\n\n\t/**\n\t * Returns {@code true} if the function is constant.\n\t * \n\t * @return {@code true} if the function is constant.\n\t */\n\tpublic boolean isConstant() {\n\t\treturn ArrayUtils.isConstant(predictions, 1, predictions.length, predictions[0])\n\t\t\t\t&& MathUtils.isZero(predictionOnMV - predictions[0]);\n\t}\n\n\t/**\n\t * Multiplies this function with a constant.\n\t * \n\t * @param c the constant.\n\t * @return this function.\n\t */\n\tpublic Function1D multiply(double c) {\n\t\tVectorUtils.multiply(predictions, c);\n\t\tpredictionOnMV *= c;\n\t\treturn this;\n\t}\n\n\t/**\n\t * Divides this function with a constant.\n\t * \n\t * @param c the constant.\n\t * @return this function.\n\t */\n\tpublic Function1D divide(double c) {\n\t\tVectorUtils.divide(predictions, c);\n\t\tpredictionOnMV /= c;\n\t\treturn this;\n\t}\n\n\t/**\n\t * Adds this function with a constant.\n\t * \n\t * @param c the constant.\n\t * @return this function.\n\t */\n\tpublic Function1D add(double c) {\n\t\tVectorUtils.add(predictions, c);\n\t\tpredictionOnMV += c;\n\t\treturn this;\n\t}\n\n\t/**\n\t * Subtracts this function with a constant.\n\t * \n\t * @param c the constant.\n\t * @return this function.\n\t */\n\tpublic Function1D subtract(double c) {\n\t\tVectorUtils.subtract(predictions, c);\n\t\tpredictionOnMV -= c;\n\t\treturn this;\n\t}\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Function1D() {\n\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param attIndex the attribute index.\n\t * @param splits the splits.\n\t * @param predictions the predictions.\n\t */\n\tpublic Function1D(int attIndex, double[] splits, double[] predictions) {\n\t\tthis(attIndex, splits, predictions, 0.0);\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param attIndex the attribute index.\n\t * @param splits the splits.\n\t * @param predictions the predictions.\n\t * @param predictionOnMissing prediction on missing value;\n\t */\n\tpublic Function1D(int attIndex, double[] splits, double[] predictions, double predictionOnMissing) {\n\t\tthis.attIndex = attIndex;\n\t\tthis.splits = splits;\n\t\tthis.predictions = predictions;\n\t\tthis.predictionOnMV = predictionOnMissing;\n\t}\n\n\t/**\n\t * Adds this function whit another function.\n\t * \n\t * @param func the other function.\n\t * @return this function.\n\t */\n\tpublic Function1D add(Function1D func) {\n\t\tif (attIndex != func.attIndex) {\n\t\t\tthrow new IllegalArgumentException(\"Cannot add functions on different terms\");\n\t\t}\n\t\tint[] insertionPoints = new int[func.splits.length - 1];\n\t\tint newElements = 0;\n\t\tfor (int i = 0; i < insertionPoints.length; i++) {\n\t\t\tinsertionPoints[i] = Arrays.binarySearch(splits, func.splits[i]);\n\t\t\tif (insertionPoints[i] < 0) {\n\t\t\t\tnewElements++;\n\t\t\t}\n\t\t}\n\t\tif (newElements > 0) {\n\t\t\tdouble[] newSplits = new double[splits.length + newElements];\n\t\t\tSystem.arraycopy(splits, 0, newSplits, 0, splits.length);\n\t\t\tint k = splits.length;\n\t\t\tfor (int i = 0; i < insertionPoints.length; i++) {\n\t\t\t\tif (insertionPoints[i] < 0) {\n\t\t\t\t\tnewSplits[k++] = func.splits[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\tArrays.sort(newSplits);\n\n\t\t\tdouble[] newPredictions = new double[newSplits.length];\n\t\t\tfor (int i = 0; i < newSplits.length; i++) {\n\t\t\t\tnewPredictions[i] = this.evaluate(newSplits[i]) + func.evaluate(newSplits[i]);\n\t\t\t}\n\t\t\tsplits = newSplits;\n\t\t\tpredictions = newPredictions;\n\t\t} else {\n\t\t\tfor (int i = 0; i < splits.length; i++) {\n\t\t\t\tpredictions[i] += func.evaluate(splits[i]);\n\t\t\t}\n\t\t}\n\t\tthis.predictionOnMV += func.predictionOnMV;\n\t\treturn this;\n\t}\n\n\t/**\n\t * Returns the attribute index.\n\t * \n\t * @return the attribute index.\n\t */\n\tpublic int getAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\n\t/**\n\t * Sets the attribute index.\n\t * \n\t * @param attIndex the new attribute index.\n\t */\n\tpublic void setAttributeIndex(int attIndex) {\n\t\tthis.attIndex = attIndex;\n\t}\n\n\t/**\n\t * Returns the internal split array.\n\t * \n\t * @return the internal split array.\n\t */\n\tpublic double[] getSplits() {\n\t\treturn splits;\n\t}\n\n\t/**\n\t * Sets the split array.\n\t * \n\t * @param splits the new split array.\n\t */\n\tpublic void setSplits(double[] splits) {\n\t\tthis.splits = splits;\n\t}\n\n\t/**\n\t * Returns the internal prediction array.\n\t * \n\t * @return the internal prediction array.\n\t */\n\tpublic double[] getPredictions() {\n\t\treturn predictions;\n\t}\n\n\t/**\n\t * Sets the prediction array.\n\t * \n\t * @param predictions the new prediction array.\n\t */\n\tpublic void setPredictions(double[] predictions) {\n\t\tthis.predictions = predictions;\n\t}\n\t\n\t/**\n\t * Returns the prediction on missing value.\n\t * \n\t * @return the prediction on missing value.\n\t */\n\tpublic double getPredictionOnMV() {\n\t\treturn predictionOnMV;\n\t}\n\t\n\t/**\n\t * Sets the prediction on missing value.\n\t * \n\t * @param predictionOnMV the prediction on missing value.\n\t */\n\tpublic void setPredictionOnMV(double predictionOnMV) {\n\t\tthis.predictionOnMV = predictionOnMV;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tString line = in.readLine();\n\t\tString[] data = line.split(\": \");\n\t\tattIndex = Integer.parseInt(data[1]);\n\n\t\tline = in.readLine();\n\t\tdata = line.split(\": \");\n\t\tpredictionOnMV = Double.parseDouble(data[1]);\n\t\t\n\t\tin.readLine();\n\t\tline = in.readLine();\n\t\tsplits = ArrayUtils.parseDoubleArray(line);\n\n\t\tin.readLine();\n\t\tline = in.readLine();\n\t\tpredictions = ArrayUtils.parseDoubleArray(line);\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"AttIndex: \" + attIndex);\n\t\tout.println(\"PredictinOnMV: \" + predictionOnMV);\n\t\tout.println(\"Splits: \" + splits.length);\n\t\tout.println(Arrays.toString(splits));\n\t\tout.println(\"Predictions: \" + predictions.length);\n\t\tout.println(Arrays.toString(predictions));\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\treturn evaluate(instance.getValue(attIndex));\n\t}\n\n\t@Override\n\tpublic double evaluate(double x) {\n\t\tif (Double.isNaN(x)) {\n\t\t\treturn predictionOnMV;\n\t\t} else {\n\t\t\treturn predictions[getSegmentIndex(x)];\n\t\t}\n\t}\n\n\t@Override\n\tpublic Function1D copy() {\n\t\tdouble[] splitsCopy = Arrays.copyOf(splits, splits.length);\n\t\tdouble[] predictionsCopy = Arrays.copyOf(predictions, predictions.length);\n\t\treturn new Function1D(attIndex, splitsCopy, predictionsCopy, predictionOnMV);\n\t}\n\t\n\t/**\n\t * Returns the segment index given x.\n\t * \n\t * @param x the search key.\n\t * @return the segment index given x.\n\t */\n\tprotected int getSegmentIndex(double x) {\n\t\t// Assume x is not NaN\n\t\treturn ArrayUtils.findInsertionPoint(splits, x);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/Function2D.java",
    "content": "package mltk.predictor.function;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\nimport java.util.Arrays;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.Regressor;\r\nimport mltk.util.ArrayUtils;\r\nimport mltk.util.VectorUtils;\r\nimport mltk.util.tuple.IntPair;\r\n\r\n/**\r\n * Class for 2D functions.\r\n * \r\n * <p>\r\n * This class represents a segmented 2D function. Segments are defined in split arrays for the two attributes. For\r\n * example, [3, 5, +INF] defines three segments: (-INF, 3], (3, 5], (5, +INF). The last value in the split array is\r\n * always +INF. The prediction matrix is the corresponding predictions for segments defined in splits.\r\n * </p>\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Function2D implements Regressor, BivariateFunction {\r\n\r\n\t/**\r\n\t * First attribute index.\r\n\t */\r\n\tprotected int attIndex1;\r\n\r\n\t/**\r\n\t * Second attribute index.\r\n\t */\r\n\tprotected int attIndex2;\r\n\t\r\n\t/**\r\n\t * Last value is always Double.POSITIVE_INFINITY. e.g. [3, 5, +INF] defines three segments: (-INF, 3], (3, 5], (5,\r\n\t * +INF)\r\n\t */\r\n\tprotected double[] splits1;\r\n\tprotected double[] splits2;\r\n\r\n\t/**\r\n\t * Predictions.\r\n\t */\r\n\tprotected double[][] predictions;\r\n\t\r\n\t/**\r\n\t * Predictions on missing value for attribute 1.\r\n\t */\r\n\tprotected double[] predictionsOnMV1;\r\n\t\r\n\t/**\r\n\t * Predictions on missing value for attribute 2.\r\n\t */\r\n\tprotected double[] predictionsOnMV2;\r\n\t\r\n\t/**\r\n\t * Prediction when both attribute 1 and 2 are missing.\r\n\t */\r\n\tprotected double predictionOnMV12;\r\n\t\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic Function2D() {\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param attIndex1 the 1st attribute index.\r\n\t * @param attIndex2 the 2nd attribute index.\r\n\t * @param splits1 the split array for the 1st attribute.\r\n\t * @param splits2 the split array for the 2nd attribute.\r\n\t * @param predictions the prediction matrix.\r\n\t */\r\n\tpublic Function2D(int attIndex1, int attIndex2, double[] splits1, double[] splits2, double[][] predictions) {\r\n\t\tthis(attIndex1, attIndex2, splits1, splits2, predictions, new double[splits2.length], new double[splits1.length], 0.0);\r\n\t}\r\n\t\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param attIndex1 the 1st attribute index.\r\n\t * @param attIndex2 the 2nd attribute index.\r\n\t * @param splits1 the split array for the 1st attribute.\r\n\t * @param splits2 the split array for the 2nd attribute.\r\n\t * @param predictions the prediction matrix.\r\n\t * @param predictionsOnMV1 the prediction array when the 1st attribute is missing.\r\n\t * @param predictionsOnMV2 the prediction array when the 2nd attribute is missing.\r\n\t * @param predictionOnMV12 the prediction when both attributes are missing.\r\n\t */\r\n\tpublic Function2D(int attIndex1, int attIndex2,\r\n\t\t\tdouble[] splits1, double[] splits2, double[][] predictions,\r\n\t\t\tdouble[] predictionsOnMV1, double[] predictionsOnMV2, double predictionOnMV12) {\r\n\t\tthis.attIndex1 = attIndex1;\r\n\t\tthis.attIndex2 = attIndex2;\r\n\t\tthis.predictions = predictions;\r\n\t\tthis.splits1 = splits1;\r\n\t\tthis.splits2 = splits2;\r\n\t\tthis.predictionsOnMV1 = predictionsOnMV1;\r\n\t\tthis.predictionsOnMV2 = predictionsOnMV2;\r\n\t\tthis.predictionOnMV12 = predictionOnMV12;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns a constant 2D function.\r\n\t * \r\n\t * @param attIndex1 the 1st attribute index.\r\n\t * @param attIndex2 the 2nd attribute index.\r\n\t * @param prediction the constant.\r\n\t * @return a constant 2D function.\r\n\t */\r\n\tpublic static Function2D getConstantFunction(int attIndex1, int attIndex2, double prediction) {\r\n\t\tFunction2D func = new Function2D(attIndex1, attIndex2, new double[] { Double.POSITIVE_INFINITY },\r\n\t\t\t\tnew double[] { Double.POSITIVE_INFINITY }, new double[][] { { prediction } });\r\n\t\treturn func;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the index of 1st attribute.\r\n\t * \r\n\t * @return the index of 1st attribute.\r\n\t */\r\n\tpublic int getAttributeIndex1() {\r\n\t\treturn attIndex1;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the index of 2nd attribute.\r\n\t * \r\n\t * @return the index of 2nd attribute.\r\n\t */\r\n\tpublic int getAttributeIndex2() {\r\n\t\treturn attIndex2;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the attribute indices pair.\r\n\t * \r\n\t * @return the attribute indices pair.\r\n\t */\r\n\tpublic IntPair getAttributeIndices() {\r\n\t\treturn new IntPair(attIndex1, attIndex2);\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the attribute indices.\r\n\t * \r\n\t * @param attIndex1 the new index for the 1st attribute.\r\n\t * @param attIndex2 the new index for the 2nd attribute.\r\n\t */\r\n\tpublic void setAttributeIndices(int attIndex1, int attIndex2) {\r\n\t\tthis.attIndex1 = attIndex1;\r\n\t\tthis.attIndex2 = attIndex2;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the internal prediction matrix.\r\n\t * \r\n\t * @return the internal prediction matrix.\r\n\t */\r\n\tpublic double[][] getPredictions() {\r\n\t\treturn predictions;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Sets the prediction matrix.\r\n\t * \r\n\t * @param predictions the new prediction matrix.\r\n\t */\r\n\tpublic void setPredictions(double[][] predictions) {\r\n\t\tthis.predictions = predictions;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the prediction array when the 1st attribute is missing.\r\n\t * \r\n\t * @return the prediction array when the 1st attribute is missing.\r\n\t */\r\n\tpublic double[] getPredictionsOnMV1() {\r\n\t\treturn predictionsOnMV1;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Sets the prediction array when the 1st attribute is missing.\r\n\t * \r\n\t * @param predictionsOnMV1 the new prediction array.\r\n\t */\r\n\tpublic void setPredictionsOnMV1(double[] predictionsOnMV1) {\r\n\t\tthis.predictionsOnMV1 = predictionsOnMV1;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the prediction array when the 2nd attribute is missing.\r\n\t * \r\n\t * @return the prediction array when the 2nd attribute is missing.\r\n\t */\r\n\tpublic double[] getPredictionsOnMV2() {\r\n\t\treturn predictionsOnMV2;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Sets the prediction array when the 2nd attribute is missing.\r\n\t * \r\n\t * @param predictionsOnMV2 the new prediction array.\r\n\t */\r\n\tpublic void setPredictionsOnMV2(double[] predictionsOnMV2) {\r\n\t\tthis.predictionsOnMV2 = predictionsOnMV2;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the prediction when both attributes are missing.\r\n\t * \r\n\t * @return the prediction when both attributes are missing.\r\n\t */\r\n\tpublic double getPredictionOnMV12() {\r\n\t\treturn predictionOnMV12;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Sets the prediction when both attributes are missing.\r\n\t * \r\n\t * @param predictionOnMV12 the new prediction.\r\n\t */\r\n\tpublic void setPredictionOnMV12(double predictionOnMV12) {\r\n\t\tthis.predictionOnMV12 = predictionOnMV12;\r\n\t}\r\n\r\n\t/**\r\n\t * Multiplies this function with a constant.\r\n\t * \r\n\t * @param c the constant.\r\n\t * @return this function.\r\n\t */\r\n\tpublic Function2D multiply(double c) {\r\n\t\tfor (double[] preds : predictions) {\r\n\t\t\tVectorUtils.multiply(preds, c);\r\n\t\t}\r\n\t\tVectorUtils.multiply(predictionsOnMV1, c);\r\n\t\tVectorUtils.multiply(predictionsOnMV2, c);\r\n\t\tpredictionOnMV12 *= c;\r\n\t\treturn this;\r\n\t}\r\n\r\n\t/**\r\n\t * Divides this function with a constant.\r\n\t * \r\n\t * @param c the constant.\r\n\t * @return this function.\r\n\t */\r\n\tpublic Function2D divide(double c) {\r\n\t\tfor (double[] preds : predictions) {\r\n\t\t\tVectorUtils.divide(preds, c);\r\n\t\t}\r\n\t\tVectorUtils.divide(predictionsOnMV1, c);\r\n\t\tVectorUtils.divide(predictionsOnMV2, c);\r\n\t\tpredictionOnMV12 /= c;\r\n\t\treturn this;\r\n\t}\r\n\r\n\t/**\r\n\t * Adds this function with a constant.\r\n\t * \r\n\t * @param c the constant.\r\n\t * @return this function.\r\n\t */\r\n\tpublic Function2D add(double c) {\r\n\t\tfor (double[] preds : predictions) {\r\n\t\t\tVectorUtils.add(preds, c);\r\n\t\t}\r\n\t\tVectorUtils.add(predictionsOnMV1, c);\r\n\t\tVectorUtils.add(predictionsOnMV2, c);\r\n\t\tpredictionOnMV12 += c;\r\n\t\treturn this;\r\n\t}\r\n\r\n\t/**\r\n\t * Subtracts this function with a constant.\r\n\t * \r\n\t * @param c the constant.\r\n\t * @return this function.\r\n\t */\r\n\tpublic Function2D subtract(double c) {\r\n\t\tfor (double[] preds : predictions) {\r\n\t\t\tVectorUtils.subtract(preds, c);\r\n\t\t}\r\n\t\tVectorUtils.subtract(predictionsOnMV1, c);\r\n\t\tVectorUtils.subtract(predictionsOnMV2, c);\r\n\t\tpredictionOnMV12 -= c;\r\n\t\treturn this;\r\n\t}\r\n\r\n\t/**\r\n\t * Adds this function with another one.\r\n\t * \r\n\t * @param func the other function.\r\n\t * @return this function.\r\n\t */\r\n\tpublic Function2D add(Function2D func) {\r\n\t\tif (attIndex1 != func.attIndex1 || attIndex2 != func.attIndex2) {\r\n\t\t\tthrow new IllegalArgumentException(\"Cannot add arrays on differnt terms\");\r\n\t\t}\r\n\t\tdouble[] s1 = splits1;\r\n\t\tint[] insertionPoints1 = new int[func.splits1.length - 1];\r\n\t\tint newElements1 = 0;\r\n\t\tfor (int i = 0; i < insertionPoints1.length; i++) {\r\n\t\t\tinsertionPoints1[i] = Arrays.binarySearch(splits1, func.splits1[i]);\r\n\t\t\tif (insertionPoints1[i] < 0) {\r\n\t\t\t\tnewElements1++;\r\n\t\t\t}\r\n\t\t}\r\n\t\tif (newElements1 > 0) {\r\n\t\t\tdouble[] newSplits1 = new double[splits1.length + newElements1];\r\n\t\t\tSystem.arraycopy(splits1, 0, newSplits1, 0, splits1.length);\r\n\t\t\tint k = splits1.length;\r\n\t\t\tfor (int i = 0; i < insertionPoints1.length; i++) {\r\n\t\t\t\tif (insertionPoints1[i] < 0) {\r\n\t\t\t\t\tnewSplits1[k++] = func.splits1[i];\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tArrays.sort(newSplits1);\r\n\t\t\ts1 = newSplits1;\r\n\t\t}\r\n\t\tdouble[] s2 = splits2;\r\n\t\tint[] insertionPoints2 = new int[func.splits2.length - 1];\r\n\t\tint newElements2 = 0;\r\n\t\tfor (int j = 0; j < insertionPoints2.length; j++) {\r\n\t\t\tinsertionPoints2[j] = Arrays.binarySearch(splits2, func.splits2[j]);\r\n\t\t\tif (insertionPoints2[j] < 0) {\r\n\t\t\t\tnewElements2++;\r\n\t\t\t}\r\n\t\t}\r\n\t\tif (newElements2 > 0) {\r\n\t\t\tdouble[] newSplits2 = new double[splits2.length + newElements2];\r\n\t\t\tSystem.arraycopy(splits2, 0, newSplits2, 0, splits2.length);\r\n\t\t\tint k = splits2.length;\r\n\t\t\tfor (int j = 0; j < insertionPoints2.length; j++) {\r\n\t\t\t\tif (insertionPoints2[j] < 0) {\r\n\t\t\t\t\tnewSplits2[k++] = func.splits2[j];\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tArrays.sort(newSplits2);\r\n\t\t\ts2 = newSplits2;\r\n\t\t}\r\n\r\n\t\tif (newElements1 == 0 && newElements2 == 0) {\r\n\t\t\tfor (int i = 0; i < splits1.length; i++) {\r\n\t\t\t\tpredictionsOnMV2[i] += func.evaluate(splits1[i], Double.NaN);\r\n\t\t\t\t\r\n\t\t\t\tdouble[] ps = predictions[i];\r\n\t\t\t\tfor (int j = 0; j < splits2.length; j++) {\r\n\t\t\t\t\tpredictionsOnMV1[j] += func.evaluate(Double.NaN, splits2[j]);\r\n\t\t\t\t\t\r\n\t\t\t\t\tps[j] += func.evaluate(splits1[i], splits2[j]);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tpredictionOnMV12 += func.predictionOnMV12;\r\n\t\t} else {\r\n\t\t\tdouble[][] newPredictions = new double[s1.length][s2.length];\r\n\t\t\tpredictionsOnMV1 = new double[s2.length];\r\n\t\t\tpredictionsOnMV2 = new double[s1.length];\r\n\t\t\tfor (int i = 0; i < s1.length; i++) {\r\n\t\t\t\tpredictionsOnMV2[i] = this.evaluate(s1[i], Double.NaN) + func.evaluate(s1[i], Double.NaN);\r\n\t\t\t\t\r\n\t\t\t\tdouble[] ps = newPredictions[i];\r\n\t\t\t\tfor (int j = 0; j < s2.length; j++) {\r\n\t\t\t\t\tpredictionsOnMV1[j] = this.evaluate(Double.NaN, s2[j]) + func.evaluate(Double.NaN, s2[j]);\r\n\t\t\t\t\t\r\n\t\t\t\t\tps[j] = this.evaluate(s1[i], s2[j]) + func.evaluate(s1[i], s2[j]);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tsplits1 = s1;\r\n\t\t\tsplits2 = s2;\r\n\t\t\tpredictions = newPredictions;\r\n\t\t\tpredictionOnMV12 = this.predictionOnMV12 + func.predictionOnMV12;\r\n\t\t}\r\n\t\t\r\n\t\treturn this;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tString line = in.readLine();\r\n\t\tString[] data = line.split(\": \");\r\n\t\tattIndex1 = Integer.parseInt(data[1]);\r\n\t\tline = in.readLine();\r\n\t\tdata = line.split(\": \");\r\n\t\tattIndex2 = Integer.parseInt(data[1]);\r\n\r\n\t\tin.readLine();\r\n\t\tline = in.readLine();\r\n\t\tsplits1 = ArrayUtils.parseDoubleArray(line);\r\n\r\n\t\tin.readLine();\r\n\t\tline = in.readLine();\r\n\t\tsplits2 = ArrayUtils.parseDoubleArray(line);\r\n\r\n\t\tString[] dim = in.readLine().split(\": \")[1].split(\"x\");\r\n\t\tpredictions = new double[Integer.parseInt(dim[0])][];\r\n\t\tfor (int i = 0; i < predictions.length; i++) {\r\n\t\t\tpredictions[i] = ArrayUtils.parseDoubleArray(in.readLine());\r\n\t\t}\r\n\t\t\r\n\t\tin.readLine();\r\n\t\tpredictionsOnMV1 = ArrayUtils.parseDoubleArray(in.readLine());\r\n\t\t\r\n\t\tin.readLine();\r\n\t\tpredictionsOnMV2 = ArrayUtils.parseDoubleArray(in.readLine());\r\n\t\t\r\n\t\tdata = in.readLine().split(\": \");\r\n\t\tpredictionOnMV12 = Double.parseDouble(data[1]);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"AttIndex1: \" + attIndex1);\r\n\t\tout.println(\"AttIndex2: \" + attIndex2);\r\n\t\tout.println(\"Splits1: \" + splits1.length);\r\n\t\tout.println(Arrays.toString(splits1));\r\n\t\tout.println(\"Splits2: \" + splits2.length);\r\n\t\tout.println(Arrays.toString(splits2));\r\n\t\tout.println(\"Predictions: \" + predictions.length + \"x\" + predictions[0].length);\r\n\t\tfor (int i = 0; i < predictions.length; i++) {\r\n\t\t\tout.println(Arrays.toString(predictions[i]));\r\n\t\t}\r\n\t\tout.println(\"PredictionsOnMV1: \" + predictionsOnMV1.length);\r\n\t\tout.println(Arrays.toString(predictionsOnMV1));\r\n\t\tout.println(\"PredictionsOnMV2: \" + predictionsOnMV2.length);\r\n\t\tout.println(Arrays.toString(predictionsOnMV2));\r\n\t\tout.println(\"PredictionOnMV12: \" + predictionOnMV12);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\treturn evaluate(instance.getValue(attIndex1), instance.getValue(attIndex2));\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double evaluate(double x, double y) {\r\n\t\tif (!Double.isNaN(x) && !Double.isNaN(y)) {\r\n\t\t\tIntPair idx = getSegmentIndex(x, y);\r\n\t\t\treturn predictions[idx.v1][idx.v2];\r\n\t\t} else if (Double.isNaN(x) && !Double.isNaN(y)) {\r\n\t\t\tint idx = ArrayUtils.findInsertionPoint(splits2, y);\r\n\t\t\treturn predictionsOnMV1[idx];\r\n\t\t} else if (!Double.isNaN(x) && Double.isNaN(y)) {\r\n\t\t\tint idx = ArrayUtils.findInsertionPoint(splits1, x);\r\n\t\t\treturn predictionsOnMV2[idx];\r\n\t\t} else {\r\n\t\t\treturn predictionOnMV12;\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic Function2D copy() {\r\n\t\tdouble[] splits1Copy = Arrays.copyOf(splits1, splits1.length);\r\n\t\tdouble[] splits2Copy = Arrays.copyOf(splits2, splits2.length);\r\n\t\tdouble[][] predictionsCopy = new double[predictions.length][];\r\n\t\tfor (int i = 0; i < predictionsCopy.length; i++) {\r\n\t\t\tpredictionsCopy[i] = Arrays.copyOf(predictions[i], predictions[i].length);\r\n\t\t}\r\n\t\tdouble[] predictionsOnMV1Copy = Arrays.copyOf(predictionsOnMV1, predictionsOnMV1.length);\r\n\t\tdouble[] predictionsOnMV2Copy = Arrays.copyOf(predictionsOnMV2, predictionsOnMV2.length);\r\n\t\treturn new Function2D(attIndex1, attIndex2, splits1Copy, splits2Copy, predictionsCopy,\r\n\t\t\t\tpredictionsOnMV1Copy, predictionsOnMV2Copy, predictionOnMV12);\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the segment indices pair given (x1, x2). Assume x1 and x2 are not missing values.\r\n\t * \r\n\t * @param x1 the 1st search key.\r\n\t * @param x2 the 2nd search key.\r\n\t * @return segment indices pair at (x1, x2).\r\n\t */\r\n\tprotected IntPair getSegmentIndex(double x1, double x2) {\r\n\t\tint idx1 = ArrayUtils.findInsertionPoint(splits1, x1);\r\n\t\tint idx2 = ArrayUtils.findInsertionPoint(splits2, x2);\r\n\t\treturn new IntPair(idx1, idx2);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/Histogram2D.java",
    "content": "package mltk.predictor.function;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.core.Instances;\r\nimport mltk.util.tuple.Pair;\r\n\r\n/**\r\n * Class for 2D histograms.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Histogram2D {\r\n\r\n\tpublic double[][] resp;\r\n\tpublic double[][] count;\r\n\tpublic double[] respOnMV1;\r\n\tpublic double[] countOnMV1;\r\n\tpublic double[] respOnMV2;\r\n\tpublic double[] countOnMV2;\r\n\tpublic double respOnMV12;\r\n\tpublic double countOnMV12;\r\n\t\r\n\tpublic static class Table {\r\n\r\n\t\tpublic double[][][] resp;\r\n\t\tpublic double[][][] count;\r\n\t\tpublic double[][] respOnMV1;\r\n\t\tpublic double[][] countOnMV1;\r\n\t\tpublic double[][] respOnMV2;\r\n\t\tpublic double[][] countOnMV2;\r\n\t\tpublic double respOnMV12;\r\n\t\tpublic double countOnMV12;\r\n\t\t\r\n\r\n\t\tpublic Table(int n, int m) {\r\n\t\t\tresp = new double[n][m][4];\r\n\t\t\tcount = new double[n][m][4];\r\n\t\t\trespOnMV1 = new double[m][2];\r\n\t\t\tcountOnMV1 = new double[m][2];\r\n\t\t\trespOnMV2 = new double[n][2];\r\n\t\t\tcountOnMV2 = new double[n][2];\r\n\t\t\trespOnMV12 = 0.0;\r\n\t\t\tcountOnMV12 = 0.0;\r\n\t\t}\r\n\t}\r\n\t\r\n\t/**\r\n\t * Computes 2D histogram given (f1, f2).\r\n\t * \r\n\t * @param instances the data set.\r\n\t * @param f1 the 1st feature.\r\n\t * @param f2 the 2nd feature.\r\n\t * @param hist2d the histogram to compute.\r\n\t */\r\n\tpublic static void computeHistogram2D(Instances instances, int f1, int f2, Histogram2D hist2d) {\r\n\t\tfor (Instance instance : instances) {\r\n\t\t\tdouble resp = instance.getTarget() * instance.getWeight();\r\n\t\t\tdouble weight = instance.getWeight();\r\n\t\t\tif (!instance.isMissing(f1) && !instance.isMissing(f2)) {\r\n\t\t\t\tint idx1 = (int) instance.getValue(f1);\r\n\t\t\t\tint idx2 = (int) instance.getValue(f2);\r\n\t\t\t\thist2d.resp[idx1][idx2] += resp;\r\n\t\t\t\thist2d.count[idx1][idx2] += weight;\r\n\t\t\t} else if (instance.isMissing(f1) && !instance.isMissing(f2)) {\r\n\t\t\t\tint idx2 = (int) instance.getValue(f2);\r\n\t\t\t\thist2d.respOnMV1[idx2] += resp;\r\n\t\t\t\thist2d.countOnMV1[idx2] += weight;\r\n\t\t\t} else if (!instance.isMissing(f1) && instance.isMissing(f2)) {\r\n\t\t\t\tint idx1 = (int) instance.getValue(f1);\r\n\t\t\t\thist2d.respOnMV2[idx1] += resp;\r\n\t\t\t\thist2d.countOnMV2[idx1] += weight;\r\n\t\t\t} else {\r\n\t\t\t\thist2d.respOnMV12 += resp;\r\n\t\t\t\thist2d.countOnMV12 += weight;\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\t\r\n\t/**\r\n\t * Computes auxiliary data structure given 2D histogram and cumulative 1D histograms.\r\n\t * \r\n\t * @param hist2d the 2D histogram.\r\n\t * @param cHist1 the cumulative histogram for the 1st feature.\r\n\t * @param cHist2 the cumulative histogram for the 2nd feature.\r\n\t * @return table auxiliary data structure.\r\n\t */\r\n\tpublic static Table computeTable(Histogram2D hist2d, CHistogram cHist1, CHistogram cHist2) {\r\n\t\tTable table = new Table(hist2d.resp.length, hist2d.resp[0].length);\r\n\t\tdouble sum = 0;\r\n\t\tdouble count = 0;\r\n\t\tfor (int j = 0; j < hist2d.resp[0].length; j++) {\r\n\t\t\tsum += hist2d.resp[0][j];\r\n\t\t\ttable.resp[0][j][0] = sum;\r\n\t\t\tcount += hist2d.count[0][j];\r\n\t\t\ttable.count[0][j][0] = count;\r\n\t\t\tfillTable(table, 0, j, cHist1, cHist2);\r\n\t\t}\r\n\t\tfor (int i = 1; i < hist2d.resp.length; i++) {\r\n\t\t\tsum = count = 0;\r\n\t\t\tfor (int j = 0; j < hist2d.resp[i].length; j++) {\r\n\t\t\t\tsum += hist2d.resp[i][j];\r\n\t\t\t\ttable.resp[i][j][0] = table.resp[i - 1][j][0] + sum;\r\n\t\t\t\tcount += hist2d.count[i][j];\r\n\t\t\t\ttable.count[i][j][0] = table.count[i - 1][j][0] + count;\r\n\t\t\t\tfillTable(table, i, j, cHist1, cHist2);\r\n\t\t\t}\r\n\t\t}\r\n\t\t\r\n\t\tdouble respOnMV1 = 0;\r\n\t\tdouble countOnMV1 = 0;\r\n\t\tfor (int j = 0; j < hist2d.respOnMV1.length; j++) {\r\n\t\t\trespOnMV1 += hist2d.respOnMV1[j];\r\n\t\t\tcountOnMV1 += hist2d.countOnMV1[j];\r\n\t\t\ttable.respOnMV1[j][0] = respOnMV1;\r\n\t\t\ttable.respOnMV1[j][1] = cHist1.sumOnMV - respOnMV1;\r\n\t\t\ttable.countOnMV1[j][0] = countOnMV1;\r\n\t\t\ttable.countOnMV1[j][1] = cHist1.countOnMV - countOnMV1;\r\n\t\t}\r\n\t\t\r\n\t\tdouble respOnMV2 = 0;\r\n\t\tdouble countOnMV2 = 0;\r\n\t\tfor (int i = 0; i < hist2d.respOnMV2.length; i++) {\r\n\t\t\trespOnMV2 += hist2d.respOnMV2[i];\r\n\t\t\tcountOnMV2 += hist2d.countOnMV2[i];\r\n\t\t\ttable.respOnMV2[i][0] = respOnMV2;\r\n\t\t\ttable.respOnMV2[i][1] = cHist2.sumOnMV - respOnMV2;\r\n\t\t\ttable.countOnMV2[i][0] = countOnMV2;\r\n\t\t\ttable.countOnMV2[i][1] = cHist2.countOnMV - countOnMV2;\r\n\t\t}\r\n\t\t\r\n\t\ttable.respOnMV12 = hist2d.respOnMV12;\r\n\t\ttable.countOnMV12 = hist2d.countOnMV12;\r\n\t\t\r\n\t\treturn table;\r\n\t}\r\n\r\n\tprotected static void fillTable(Table table, int i, int j, CHistogram cHist1, CHistogram cHist2) {\r\n\t\tdouble[] count = table.count[i][j];\r\n\t\tdouble[] resp = table.resp[i][j];\r\n\t\tresp[1] = cHist1.sum[i] - resp[0];\r\n\t\tresp[2] = cHist2.sum[j] - resp[0];\r\n\t\tresp[3] = cHist1.sum[cHist1.size() - 1] - cHist1.sum[i] - resp[2];\r\n\r\n\t\tcount[1] = cHist1.count[i] - count[0];\r\n\t\tcount[2] = cHist2.count[j] - count[0];\r\n\t\tcount[3] = cHist1.count[cHist1.size() - 1] - cHist1.count[i] - count[2];\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param n the size of the 1st dimension.\r\n\t * @param m the size of the 2nd dimension.\r\n\t */\r\n\tpublic Histogram2D(int n, int m) {\r\n\t\tresp = new double[n][m];\r\n\t\tcount = new double[n][m];\r\n\t\trespOnMV1 = new double[m];\r\n\t\tcountOnMV1 = new double[m];\r\n\t\trespOnMV2 = new double[n];\r\n\t\tcountOnMV2 = new double[n];\r\n\t\trespOnMV12 = 0.0;\r\n\t\tcountOnMV12 = 0.0;\r\n\t}\r\n\r\n\t/**\r\n\t * Computes the cumulative histograms on the margin.\r\n\t * \r\n\t * @return the cumulative histograms.\r\n\t */\r\n\tpublic Pair<CHistogram, CHistogram> computeCHistogram() {\r\n\t\tCHistogram cHist1 = new CHistogram(resp.length);\r\n\t\tCHistogram cHist2 = new CHistogram(resp[0].length);\r\n\r\n\t\tfor (int i = 0; i < resp.length; i++) {\r\n\t\t\tdouble[] r = resp[i];\r\n\t\t\tdouble[] c = count[i];\r\n\t\t\tfor (int j = 0; j < r.length; j++) {\r\n\t\t\t\tcHist1.sum[i] += r[j];\r\n\t\t\t\tcHist1.count[i] += c[j];\r\n\t\t\t\t\r\n\t\t\t\tcHist2.sum[j] += r[j];\r\n\t\t\t\tcHist2.count[j] += c[j];\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tfor (int i = 1; i < cHist1.size(); i++) {\r\n\t\t\tcHist1.sum[i] += cHist1.sum[i - 1];\r\n\t\t\tcHist1.count[i] += cHist1.count[i - 1];\r\n\t\t}\r\n\r\n\t\tfor (int j = 1; j < cHist2.size(); j++) {\r\n\t\t\tcHist2.sum[j] += cHist2.sum[j - 1];\r\n\t\t\tcHist2.count[j] += cHist2.count[j - 1];\r\n\t\t}\r\n\t\t\r\n\t\tfor (int j = 0; j < respOnMV1.length; j++) {\r\n\t\t\tcHist1.sumOnMV += respOnMV1[j];\r\n\t\t\tcHist1.countOnMV += countOnMV1[j];\r\n\t\t}\r\n\t\tcHist1.sumOnMV += respOnMV12;\r\n\t\tcHist1.countOnMV += countOnMV12;\r\n\t\t\r\n\t\tfor (int i = 0; i < respOnMV2.length; i++) {\r\n\t\t\tcHist2.sumOnMV += respOnMV2[i];\r\n\t\t\tcHist2.countOnMV += countOnMV2[i];\r\n\t\t}\r\n\t\tcHist2.sumOnMV += respOnMV12;\r\n\t\tcHist2.countOnMV += countOnMV12;\r\n\r\n\t\treturn new Pair<CHistogram, CHistogram>(cHist1, cHist2);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/LineCutter.java",
    "content": "package mltk.predictor.function;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.PriorityQueue;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.predictor.Learner;\nimport mltk.util.Random;\nimport mltk.util.Element;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\n\n/**\n * Class for cutting lines.\n * \n * @author Yin Lou\n *\n */\npublic class LineCutter extends Learner {\n\n\tstatic class Interval implements Comparable<Interval> {\n\n\t\tboolean finalized;\n\t\tint start;\n\t\tint end;\n\t\t// INF: Can split but split point not computed\n\t\t// NaN: Declared as leaf\n\t\t// Other: Split point\n\t\tdouble split;\n\t\tdouble sum;\n\t\tdouble weight;\n\t\tdouble value; // mean * sum, or sum * sum /weight; negative gain\n\t\tdouble gain;\n\t\tInterval left;\n\t\tInterval right;\n\n\t\tInterval() {\n\t\t\tsplit = Double.POSITIVE_INFINITY;\n\t\t}\n\n\t\tInterval(int start, int end, double sum, double weight) {\n\t\t\tthis.start = start;\n\t\t\tthis.end = end;\n\t\t\tthis.split = Double.POSITIVE_INFINITY;\n\t\t\tthis.sum = sum;\n\t\t\tthis.weight = weight;\n\t\t}\n\n\t\t@Override\n\t\tpublic int compareTo(Interval o) {\n\t\t\tif (this.value < o.value) {\n\t\t\t\treturn -1;\n\t\t\t} else if (this.value > o.value) {\n\t\t\t\treturn 1;\n\t\t\t} else {\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t}\n\n\t\tdouble getPrediction() {\n\t\t\treturn MathUtils.divide(sum, weight, 0.0);\n\t\t}\n\n\t\tboolean isFinalized() {\n\t\t\treturn finalized;\n\t\t}\n\n\t\tboolean isInteriorNode() {\n\t\t\treturn split < Double.POSITIVE_INFINITY;\n\t\t}\n\n\t\tboolean isLeaf() {\n\t\t\treturn Double.isNaN(split);\n\t\t}\n\n\t}\n\n\tprivate int attIndex;\n\n\tprivate int numIntervals;\n\n\tprivate boolean isClassification;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LineCutter() {\n\t\tthis(false);\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param isClassification {@code true} if it is a classification problem.\n\t */\n\tpublic LineCutter(boolean isClassification) {\n\t\tattIndex = -1;\n\t\tthis.isClassification = isClassification;\n\t}\n\n\t@Override\n\tpublic Function1D build(Instances instances) {\n\t\treturn build(instances, attIndex, numIntervals);\n\t}\n\n\t/**\n\t * Builds a 1D function.\n\t * \n\t * @param instances the training set.\n\t * @param attribute the attribute.\n\t * @param numIntervals the number of intervals.\n\t * @return a 1D function.\n\t */\n\tpublic Function1D build(Instances instances, Attribute attribute, int numIntervals) {\n\t\tint attIndex = attribute.getIndex();\n\t\t\n\t\tdouble sumRespOnMV = 0.0;\n\t\tdouble sumWeightOnMV = 0.0;\n\n\t\tList<double[]> histograms;\n\t\tif (attribute.getType() == Attribute.Type.NUMERIC) {\n\t\t\t// weight: attribute value\n\t\t\t// [feature value, sum, weight]\n\t\t\tList<Element<double[]>> pairs = new ArrayList<>(instances.size());\n\t\t\tfor (Instance instance : instances) {\n\t\t\t\tdouble weight = instance.getWeight();\n\t\t\t\tdouble value = instance.getValue(attIndex);\n\t\t\t\tdouble target = instance.getTarget();\n\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\tpairs.add(new Element<>(new double[] { target, weight }, value));\n\t\t\t\t\t} else {\n\t\t\t\t\t\tpairs.add(new Element<>(new double[] { target * weight, weight }, value));\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\tsumRespOnMV += target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsumRespOnMV += target * weight;\n\t\t\t\t\t}\n\t\t\t\t\tsumWeightOnMV += weight;\n\t\t\t\t}\n\t\t\t}\n\t\t\tCollections.sort(pairs);\n\n\t\t\thistograms = new ArrayList<>(pairs.size() + 1);\n\t\t\tgetHistograms(pairs, histograms);\n\t\t\thistograms.add(new double[] { Double.NaN, sumRespOnMV, sumWeightOnMV });\n\t\t} else {\n\t\t\tint size = 0;\n\t\t\tif (attribute.getType() == Attribute.Type.BINNED) {\n\t\t\t\tsize = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t} else {\n\t\t\t\tsize = ((NominalAttribute) attribute).getCardinality();\n\t\t\t}\n\t\t\tdouble[][] histogram = new double[size][2];\n\t\t\tfor (Instance instance : instances) {\n\t\t\t\tdouble weight = instance.getWeight();\n\t\t\t\tdouble value = instance.getValue(attIndex);\n\t\t\t\tdouble target = instance.getTarget();\n\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\tint idx = (int) value;\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\thistogram[idx][0] += target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\thistogram[idx][0] += target * weight;\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\thistogram[idx][1] += weight;\n\t\t\t\t} else {\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\tsumRespOnMV += target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsumRespOnMV += target * weight;\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tsumWeightOnMV += weight;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\thistograms = new ArrayList<>(histogram.length + 1);\n\t\t\tfor (int i = 0; i < histogram.length; i++) {\n\t\t\t\tif (!MathUtils.isZero(histogram[i][1])) {\n\t\t\t\t\tdouble[] hist = histogram[i];\n\t\t\t\t\thistograms.add(new double[] {i, hist[0], hist[1]});\n\t\t\t\t}\n\t\t\t}\n\t\t\thistograms.add(new double[] { Double.NaN, sumRespOnMV, sumWeightOnMV });\n\t\t}\n\t\t\n\t\treturn build(attIndex, histograms, numIntervals);\n\t}\n\n\t/**\n\t * Builds a 1D function.\n\t * \n\t * @param instances the training set.\n\t * @param attIndex the index in the attribute list of the training set.\n\t * @param numIntervals the number of intervals.\n\t * @return a 1D function.\n\t */\n\tpublic Function1D build(Instances instances, int attIndex, int numIntervals) {\n\t\tAttribute attribute = instances.getAttributes().get(attIndex);\n\t\treturn build(instances, attribute, numIntervals);\n\t}\n\n\t/**\n\t * Returns the index in the attribute list of the training set.\n\t * \n\t * @return the index in the attribute list of the training set.\n\t */\n\tpublic int getAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\t\n\t/**\n\t * Sets the index in the attribute list of the training set.\n\t * \n\t * @param attIndex the attribute index.\n\t */\n\tpublic void setAttributeIndex(int attIndex) {\n\t\tthis.attIndex = attIndex;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if it is a classification problem.\n\t * \n\t * @return {@code true} if it is a classification problem.\n\t */\n\tpublic boolean isClassification() {\n\t\treturn isClassification;\n\t}\n\t\n\t/**\n\t * Sets {@code true} if it is a classification problem.\n\t * \n\t * @param isClassification {@code true} if it is a classification problem.\n\t */\n\tpublic void setClassification(boolean isClassification) {\n\t\tthis.isClassification = isClassification;\n\t}\n\n\t/**\n\t * Returns the number of intervals.\n\t * \n\t * @return the number of intervals.\n\t */\n\tpublic int getNumIntervals() {\n\t\treturn numIntervals;\n\t}\n\n\t/**\n\t * Sets the number of intervals.\n\t * \n\t * @param numIntervals the number of intervals.\n\t */\n\tpublic void setNumIntervals(int numIntervals) {\n\t\tthis.numIntervals = numIntervals;\n\t}\n\t\n\tprotected static void getHistograms(List<Element<double[]>> pairs, List<double[]> histograms) {\n\t\tif (pairs.size() == 0) {\n\t\t\treturn;\n\t\t}\n\t\t// Element(new double[] {sum, weight}, value)\n\t\tdouble[] hist = pairs.get(0).element;\n\t\tdouble lastValue = pairs.get(0).weight;\n\t\tdouble sum = hist[0];\n\t\tdouble weight = hist[1];\n\t\tfor (int i = 1; i < pairs.size(); i++) {\n\t\t\tElement<double[]> element = pairs.get(i);\n\t\t\thist = element.element;\n\t\t\tdouble value = element.weight;\n\t\t\tdouble s = hist[0];\n\t\t\tdouble w = hist[1];\n\t\t\tif (value != lastValue) {\n\t\t\t\thistograms.add(new double[] { lastValue, sum, weight });\n\t\t\t\tlastValue = value;\n\t\t\t\tsum = s;\n\t\t\t\tweight = w;\n\t\t\t} else {\n\t\t\t\tsum += s;\n\t\t\t\tweight += w;\n\t\t\t}\n\t\t}\n\t\thistograms.add(new double[] { lastValue, sum, weight });\n\t}\n\t\n\tprotected static Function1D build(int attIndex, List<double[]> histograms, int numIntervals) {\n\t\tFunction1D func = new Function1D();\n\t\tfunc.attIndex = attIndex;\n\t\t// [feature value, sum, weight]\n\t\tdouble[] histOnMV = histograms.get(histograms.size() - 1);\n\t\tfunc.predictionOnMV = MathUtils.divide(histOnMV[1], histOnMV[2], 0.0);\n\t\t\n\t\t// 1. Check basic leaf conditions\n\t\tif (histograms.size() <= 2) {\n\t\t\tfunc.splits = new double[] { Double.POSITIVE_INFINITY };\n\t\t\tdouble prediction = 0.0;\n\t\t\tif (histograms.size() == 2) {\n\t\t\t\tdouble[] hist = histograms.get(0);\n\t\t\t\tprediction = MathUtils.divide(hist[1], hist[2], 0.0);\n\t\t\t}\n\t\t\tfunc.predictions = new double[] { prediction };\n\t\t\treturn func;\n\t\t}\n\n\t\t// 2. Cut the line\n\t\t// 2.1 First cut\n\t\tdouble[] stats = sumUp(histograms, 0, histograms.size() - 1);\n\t\tInterval root = new Interval(0, histograms.size() - 1, stats[0], stats[1]);\n\t\tsplit(histograms, root);\n\n\t\tif (numIntervals == 2) {\n\t\t\tfunc.splits = new double[] { root.split, Double.POSITIVE_INFINITY };\n\t\t\tfunc.predictions = new double[] { root.left.getPrediction(), root.right.getPrediction() };\n\t\t} else if (numIntervals > 2) {\n\t\t\tPriorityQueue<Interval> q = new PriorityQueue<>();\n\t\t\tif (!root.isLeaf()) {\n\t\t\t\tq.add(root);\n\t\t\t}\n\n\t\t\tint numSplits = 0;\n\t\t\twhile (!q.isEmpty()) {\n\t\t\t\tInterval parent = q.remove();\n\t\t\t\tparent.finalized = true;\n\t\t\t\tsplit(histograms, parent.left);\n\t\t\t\tsplit(histograms, parent.right);\n\n\t\t\t\tif (!parent.left.isLeaf()) {\n\t\t\t\t\tq.add(parent.left);\n\t\t\t\t}\n\t\t\t\tif (!parent.right.isLeaf()) {\n\t\t\t\t\tq.add(parent.right);\n\t\t\t\t}\n\n\t\t\t\tnumSplits++;\n\t\t\t\tif (numSplits >= numIntervals - 1) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tList<Double> splits = new ArrayList<>(numIntervals - 1);\n\t\t\tList<Double> predictions = new ArrayList<>(numIntervals);\n\t\t\tinorder(root, splits, predictions);\n\t\t\tfunc.splits = new double[predictions.size()];\n\t\t\tfunc.predictions = new double[predictions.size()];\n\t\t\tfor (int i = 0; i < func.predictions.length; i++) {\n\t\t\t\tfunc.predictions[i] = predictions.get(i);\n\t\t\t}\n\t\t\tfor (int i = 0; i < func.splits.length - 1; i++) {\n\t\t\t\tfunc.splits[i] = splits.get(i);\n\t\t\t}\n\t\t\tfunc.splits[func.splits.length - 1] = Double.POSITIVE_INFINITY;\n\t\t}\n\t\treturn func;\n\t}\n\t\n\tprotected static double[] sumUp(List<double[]> histograms, int start, int end) {\n\t\tdouble sum = 0;\n\t\tdouble weight = 0;\n\t\tfor (int i = start; i < end; i++) {\n\t\t\tdouble[] hist = histograms.get(i);\n\t\t\tsum += hist[1];\n\t\t\tweight += hist[2];\n\t\t}\n\t\treturn new double[] { sum, weight };\n\t}\n\n\tprotected static void split(List<double[]> histograms, Interval parent) {\n\t\tsplit(histograms, parent, 5);\n\t}\n\n\tprotected static void split(List<double[]> histograms, Interval parent, double limit) {\n\t\t// Test if we need to split\n\t\tif (parent.weight <= limit || parent.end - parent.start <= 1) {\n\t\t\tparent.split = Double.NaN; // Declared as leaf\n\t\t} else {\n\t\t\tparent.left = new Interval();\n\t\t\tparent.right = new Interval();\n\t\t\tint start = parent.left.start = parent.start;\n\t\t\tint end = parent.right.end = parent.end;\n\t\t\tfinal double sum = parent.sum;\n\t\t\tfinal double totalWeight = parent.weight;\n\n\t\t\tdouble sum1 = histograms.get(start)[1];\n\t\t\tdouble sum2 = sum - sum1;\n\t\t\tdouble weight1 = histograms.get(start)[2];\n\t\t\tdouble weight2 = totalWeight - weight1;\n\n\t\t\tdouble bestEval = -(OptimUtils.getGain(sum1, weight1) + OptimUtils.getGain(sum2, weight2));\n\t\t\tList<double[]> splits = new ArrayList<>();\n\t\t\tsplits.add(new double[] { (histograms.get(start)[0] + histograms.get(start + 1)[0]) / 2, start, sum1,\n\t\t\t\t\tweight1, sum2, weight2 });\n\t\t\tfor (int i = start + 1; i < end - 1; i++) {\n\t\t\t\tdouble[] hist = histograms.get(i);\n\t\t\t\tfinal double s = hist[1];\n\t\t\t\tfinal double w = hist[2];\n\t\t\t\tsum1 += s;\n\t\t\t\tsum2 -= s;\n\t\t\t\tweight1 += w;\n\t\t\t\tweight2 -= w;\n\t\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\t\tif (eval <= bestEval) {\n\t\t\t\t\tdouble split = (histograms.get(i)[0] + histograms.get(i + 1)[0]) / 2;\n\t\t\t\t\tif (eval < bestEval) {\n\t\t\t\t\t\tbestEval = eval;\n\t\t\t\t\t\tsplits.clear();\n\t\t\t\t\t}\n\t\t\t\t\tsplits.add(new double[] { split, i, sum1, weight1, sum2, weight2 });\n\t\t\t\t}\n\t\t\t}\n\t\t\tRandom rand = Random.getInstance();\n\t\t\tdouble[] split = splits.get(rand.nextInt(splits.size()));\n\t\t\tparent.split = split[0];\n\t\t\tparent.left.end = (int) split[1] + 1;\n\t\t\tparent.right.start = (int) split[1] + 1;\n\t\t\tparent.left.sum = split[2];\n\t\t\tparent.left.weight = split[3];\n\t\t\tparent.right.sum = split[4];\n\t\t\tparent.right.weight = split[5];\n\t\t\tparent.gain = OptimUtils.getGain(parent.left.sum, parent.left.weight)\n\t\t\t\t\t+ OptimUtils.getGain(parent.right.sum, parent.right.weight);\n\t\t\tparent.value = -parent.gain + (sum / totalWeight * sum);\n\t\t}\n\t}\n\t\n\tprotected static void inorder(Interval parent, List<Double> splits, List<Double> predictions) {\n\t\tif (parent.isFinalized()) {\n\t\t\tinorder(parent.left, splits, predictions);\n\t\t\tsplits.add(parent.split);\n\t\t\tinorder(parent.right, splits, predictions);\n\t\t} else {\n\t\t\tpredictions.add(parent.getPrediction());\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/LinearFunction.java",
    "content": "package mltk.predictor.function;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.Regressor;\r\n\r\n/**\r\n * Class for linear functions.\r\n * \r\n * @author Yin Lou\r\n *\r\n */\r\npublic class LinearFunction implements Regressor, UnivariateFunction {\r\n\r\n\t/**\r\n\t * The attribute index.\r\n\t */\r\n\tprotected int attIndex;\r\n\r\n\t/**\r\n\t * The slope.\r\n\t */\r\n\tprotected double beta;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic LinearFunction() {\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Constructs a linear function with a provided slope value.\r\n\t * \r\n\t * @param beta the slope.\r\n\t */\r\n\tpublic LinearFunction(double beta) {\r\n\t\tthis(-1, beta);\r\n\t}\r\n\r\n\t/**\r\n\t * Constructs a linear function with a provided slope value and attribute index.\r\n\t * \r\n\t * @param attIndex the attribute index.\r\n\t * @param beta the slope.\r\n\t */\r\n\tpublic LinearFunction(int attIndex, double beta) {\r\n\t\tthis.attIndex = attIndex;\r\n\t\tthis.beta = beta;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tattIndex = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\tbeta = Double.parseDouble(in.readLine().split(\": \")[1]);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"AttIndex: \" + attIndex);\r\n\t\tout.println(\"Beta: \" + beta);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double evaluate(double x) {\r\n\t\treturn beta * x;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\treturn evaluate(instance.getValue(attIndex));\r\n\t}\r\n\r\n\tpublic double getSlope() {\r\n\t\treturn beta;\r\n\t}\r\n\r\n\tpublic void setSlope(double beta) {\r\n\t\tthis.beta = beta;\r\n\t}\r\n\r\n\tpublic int getAttributeIndex() {\r\n\t\treturn attIndex;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic LinearFunction copy() {\r\n\t\treturn new LinearFunction(attIndex, beta);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/SquareCutter.java",
    "content": "package mltk.predictor.function;\n\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.predictor.Learner;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.tuple.Pair;\n\n/**\n * Class for cutting squares.\n * \n * @author Yin Lou\n *\n */\npublic class SquareCutter extends Learner {\n\n\tprivate int attIndex1;\n\tprivate int attIndex2;\n\tprivate boolean lineSearch;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic SquareCutter() {\n\n\t}\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param lineSearch {@code true} if line search is performed in the end.\n\t */\n\tpublic SquareCutter(boolean lineSearch) {\n\t\tthis.lineSearch = lineSearch;\n\t}\n\n\t/**\n\t * Sets the attribute indices.\n\t * \n\t * @param attIndex1 the 1st index of attribute.\n\t * @param attIndex2 the 2nd index of attribute.\n\t */\n\tpublic void setAttIndices(int attIndex1, int attIndex2) {\n\t\tthis.attIndex1 = attIndex1;\n\t\tthis.attIndex2 = attIndex2;\n\t}\n\n\tpublic Function2D build(Instances instances) {\n\t\t// Note: Currently only support cutting on binned/nominal features\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tint size1 = 0;\n\t\tAttribute f1 = attributes.get(attIndex1);\n\t\tif (f1.getType() == Attribute.Type.BINNED) {\n\t\t\tsize1 = ((BinnedAttribute) f1).getNumBins();\n\t\t} else if (f1.getType() == Attribute.Type.NOMINAL) {\n\t\t\tsize1 = ((NominalAttribute) f1).getCardinality();\n\t\t}\n\t\tint size2 = 0;\n\t\tAttribute f2 = attributes.get(attIndex2);\n\t\tif (f2.getType() == Attribute.Type.BINNED) {\n\t\t\tsize2 = ((BinnedAttribute) f2).getNumBins();\n\t\t} else if (f2.getType() == Attribute.Type.NOMINAL) {\n\t\t\tsize2 = ((NominalAttribute) f2).getCardinality();\n\t\t}\n\t\tHistogram2D hist2d = new Histogram2D(size1, size2);\n\t\tHistogram2D.computeHistogram2D(instances, f1.getIndex(), f2.getIndex(), hist2d);\n\t\tPair<CHistogram, CHistogram> cHist = hist2d.computeCHistogram();\n\t\t\n\t\tif ((size1 == 1 && !cHist.v1.hasMissingValue()) || (size2 == 1 && !cHist.v2.hasMissingValue())) {\n\t\t\t// Not an interaction\n\t\t\t// Recommend: Use LineCutter to shape the non-trivial attribute\n\t\t\treturn new Function2D(f1.getIndex(), f2.getIndex(), new double[] { Double.POSITIVE_INFINITY },\n\t\t\t\t\tnew double[] { Double.POSITIVE_INFINITY }, new double[1][1]);\n\t\t}\n\t\t\n\t\tHistogram2D.Table table = Histogram2D.computeTable(hist2d, cHist.v1, cHist.v2);\n\n\t\tdouble bestRSS = Double.POSITIVE_INFINITY;\n\t\tdouble[] predInt1 = new double[9];\n\t\tint bestV1 = -1;\n\t\tint[] bestV2s = new int[3];\n\t\tint[] v2s = new int[3];\n\t\tv2s[2] = -1;\n\t\tfor (int v1 = 0; v1 < size1 - 1; v1++) {\n\t\t\tfindCuts(table, v1, v2s, cHist.v1.hasMissingValue());\n\t\t\tgetPredictor(table, v1, v2s, predInt1);\n\t\t\tdouble rss = getRSS(table, v1, v2s, predInt1);\n\t\t\tif (rss < bestRSS) {\n\t\t\t\tbestRSS = rss;\n\t\t\t\tbestV1 = v1;\n\t\t\t\tbestV2s[0] = v2s[0];\n\t\t\t\tbestV2s[1] = v2s[1];\n\t\t\t\tbestV2s[2] = v2s[2];\n\t\t\t}\n\t\t}\n\n\t\tboolean cutOnAttr2 = false;\n\n\t\tdouble[] predInt2 = new double[9];\n\t\tint[] bestV1s = new int[3];\n\t\tint bestV2 = -1;\n\t\tint[] v1s = new int[3];\n\t\tv1s[2] = -1;\n\t\tfor (int v2 = 0; v2 < size2 - 1; v2++) {\n\t\t\tfindCuts(table, v1s, v2, cHist.v2.hasMissingValue());\n\t\t\tgetPredictor(table, v1s, v2, predInt2);\n\t\t\tdouble rss = getRSS(table, v1s, v2, predInt2);\n\t\t\tif (rss < bestRSS) {\n\t\t\t\tbestRSS = rss;\n\t\t\t\tbestV2 = v2;\n\t\t\t\tbestV1s[0] = v1s[0];\n\t\t\t\tbestV1s[1] = v1s[1];\n\t\t\t\tbestV1s[2] = v1s[2];\n\t\t\t\tcutOnAttr2 = true;\n\t\t\t}\n\t\t}\n\n\t\tif (cutOnAttr2) {\n\t\t\t// Root cut on attribute 2 is better\n\t\t\tgetPredictor(table, bestV1s, bestV2, predInt2);\n\t\t\tif (lineSearch) {\n\t\t\t\tlineSearch(instances, f2.getIndex(), f1.getIndex(), bestV2, bestV1s[0], bestV1s[1], bestV1s[2], predInt2);\n\t\t\t}\n\t\t\treturn getFunction2D(f1.getIndex(), f2.getIndex(), bestV1s, bestV2, predInt2);\n\t\t} else {\n\t\t\t// Root cut on attribute 1 is better\n\t\t\tgetPredictor(table, bestV1, bestV2s, predInt1);\n\t\t\tif (lineSearch) {\n\t\t\t\tlineSearch(instances, f1.getIndex(), f2.getIndex(), bestV1, bestV2s[0], bestV2s[1], bestV2s[2], predInt1);\n\t\t\t}\n\t\t\treturn getFunction2D(f1.getIndex(), f2.getIndex(), bestV1, bestV2s, predInt1);\n\t\t}\n\t}\n\n\tprotected static void findCuts(Histogram2D.Table table, int v1, int[] v2, boolean hasMissingValue) {\n\t\t// Find upper cut\n\t\tdouble bestEval = Double.POSITIVE_INFINITY;\n\t\tfor (int i = 0; i < table.resp[v1].length - 1; i++) {\n\t\t\tdouble[] resp = table.resp[v1][i];\n\t\t\tdouble[] count = table.count[v1][i];\n\t\t\tdouble sum1 = resp[0];\n\t\t\tdouble sum2 = resp[1];\n\t\t\tdouble weight1 = count[0];\n\t\t\tdouble weight2 = count[1];\n\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\tif (eval < bestEval) {\n\t\t\t\tbestEval = eval;\n\t\t\t\tv2[0] = i;\n\t\t\t}\n\t\t}\n\n\t\t// Find lower cut\n\t\tbestEval = Double.POSITIVE_INFINITY;\n\t\tfor (int i = 0; i < table.resp[v1].length - 1; i++) {\n\t\t\tdouble[] resp = table.resp[v1][i];\n\t\t\tdouble[] count = table.count[v1][i];\n\t\t\tdouble sum1 = resp[2];\n\t\t\tdouble sum2 = resp[3];\n\t\t\tdouble weight1 = count[2];\n\t\t\tdouble weight2 = count[3];\n\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\tif (eval < bestEval) {\n\t\t\t\tbestEval = eval;\n\t\t\t\tv2[1] = i;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (hasMissingValue) {\n\t\t\t// Find cut on missing value\n\t\t\tbestEval = Double.POSITIVE_INFINITY;\n\t\t\tfor (int i = 0; i < table.respOnMV1.length; i++) {\n\t\t\t\tdouble[] respOnMV1 = table.respOnMV1[i];\n\t\t\t\tdouble[] countOnMV1 = table.countOnMV1[i];\n\t\t\t\tdouble sum1 = respOnMV1[0];\n\t\t\t\tdouble sum2 = respOnMV1[1];\n\t\t\t\tdouble weight1 = countOnMV1[0];\n\t\t\t\tdouble weight2 = countOnMV1[1];\n\t\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\t\tif (eval < bestEval) {\n\t\t\t\t\tbestEval = eval;\n\t\t\t\t\tv2[2] = i;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected static void findCuts(Histogram2D.Table table, int[] v1, int v2, boolean hasMissingValue) {\n\t\t// Find left cut\n\t\tdouble bestEval = Double.POSITIVE_INFINITY;\n\t\tfor (int i = 0; i < table.resp.length - 1; i++) {\n\t\t\tdouble[] resp = table.resp[i][v2];\n\t\t\tdouble[] count = table.count[i][v2];\n\t\t\tdouble sum1 = resp[0];\n\t\t\tdouble sum2 = resp[2];\n\t\t\tdouble weight1 = count[0];\n\t\t\tdouble weight2 = count[2];\n\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\tif (eval < bestEval) {\n\t\t\t\tbestEval = eval;\n\t\t\t\tv1[0] = i;\n\t\t\t}\n\t\t}\n\n\t\t// Find right cut\n\t\tbestEval = Double.POSITIVE_INFINITY;\n\t\tfor (int i = 0; i < table.resp.length - 1; i++) {\n\t\t\tdouble[] resp = table.resp[i][v2];\n\t\t\tdouble[] count = table.count[i][v2];\n\t\t\tdouble sum1 = resp[1];\n\t\t\tdouble sum2 = resp[3];\n\t\t\tdouble weight1 = count[1];\n\t\t\tdouble weight2 = count[3];\n\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\tif (eval < bestEval) {\n\t\t\t\tbestEval = eval;\n\t\t\t\tv1[1] = i;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (hasMissingValue) {\n\t\t\t// Find cut on missing value\n\t\t\tbestEval = Double.POSITIVE_INFINITY;\n\t\t\tfor (int i = 0; i < table.respOnMV2.length; i++) {\n\t\t\t\tdouble[] respOnMV2 = table.respOnMV2[i];\n\t\t\t\tdouble[] countOnMV2 = table.countOnMV2[i];\n\t\t\t\tdouble sum1 = respOnMV2[0];\n\t\t\t\tdouble sum2 = respOnMV2[1];\n\t\t\t\tdouble weight1 = countOnMV2[0];\n\t\t\t\tdouble weight2 = countOnMV2[1];\n\t\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\n\t\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\n\t\t\t\tdouble eval = -(eval1 + eval2);\n\t\t\t\tif (eval < bestEval) {\n\t\t\t\t\tbestEval = eval;\n\t\t\t\t\tv1[2] = i;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected static void getPredictor(Histogram2D.Table table, int v1, int[] v2, double[] pred) {\n\t\tint v21 = v2[0];\n\t\tint v22 = v2[1];\n\t\tint vMV = v2[2];\n\t\tdouble[] resp1 = table.resp[v1][v21];\n\t\tdouble[] count1 = table.count[v1][v21];\n\t\tdouble[] resp2 = table.resp[v1][v22];\n\t\tdouble[] count2 = table.count[v1][v22];\n\t\tpred[0] = MathUtils.divide(resp1[0], count1[0], 0);\n\t\tpred[1] = MathUtils.divide(resp1[1], count1[1], 0);\n\t\tpred[2] = MathUtils.divide(resp2[2], count2[2], 0);\n\t\tpred[3] = MathUtils.divide(resp2[3], count2[3], 0);\n\t\t\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] respOnMV1 = table.respOnMV1[vMV];\n\t\t\tdouble[] countOnMV1 = table.countOnMV1[vMV];\n\t\t\tpred[4] = MathUtils.divide(respOnMV1[0], countOnMV1[0], 0);\n\t\t\tpred[5] = MathUtils.divide(respOnMV1[1], countOnMV1[1], 0);\n\t\t}\n\t\t\n\t\tdouble[] respOnMV2 = table.respOnMV2[v1];\n\t\tdouble[] countOnMV2 = table.countOnMV2[v1];\n\t\tpred[6] = MathUtils.divide(respOnMV2[0], countOnMV2[0], 0);\n\t\tpred[7] = MathUtils.divide(respOnMV2[1], countOnMV2[1], 0);\n\t\t\n\t\tpred[8] = MathUtils.divide(table.respOnMV12, table.countOnMV12, 0);\n\t}\n\n\tprotected static void getPredictor(Histogram2D.Table table, int[] v1, int v2, double[] pred) {\n\t\tint v11 = v1[0];\n\t\tint v12 = v1[1];\n\t\tint vMV = v1[2];\n\t\tdouble[] resp1 = table.resp[v11][v2];\n\t\tdouble[] count1 = table.count[v11][v2];\n\t\tdouble[] resp2 = table.resp[v12][v2];\n\t\tdouble[] count2 = table.count[v12][v2];\n\t\tpred[0] = MathUtils.divide(resp1[0], count1[0], 0);\n\t\tpred[1] = MathUtils.divide(resp1[2], count1[2], 0);\n\t\tpred[2] = MathUtils.divide(resp2[1], count2[1], 0);\n\t\tpred[3] = MathUtils.divide(resp2[3], count2[3], 0);\n\t\t\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] respOnMV2 = table.respOnMV2[vMV];\n\t\t\tdouble[] countOnMV2 = table.countOnMV2[vMV];\n\t\t\tpred[4] = MathUtils.divide(respOnMV2[0], countOnMV2[0], 0);\n\t\t\tpred[5] = MathUtils.divide(respOnMV2[1], countOnMV2[1], 0);\n\t\t}\n\t\tdouble[] respOnMV1 = table.respOnMV1[v2];\n\t\tdouble[] countOnMV1 = table.countOnMV1[v2];\n\t\tpred[6] = MathUtils.divide(respOnMV1[0], countOnMV1[0], 0);\n\t\tpred[7] = MathUtils.divide(respOnMV1[1], countOnMV1[1], 0);\n\t\t\n\t\tpred[8] = MathUtils.divide(table.respOnMV12, table.countOnMV12, 0);\n\t\t\n\t}\n\n\tprotected static double getRSS(Histogram2D.Table table, int v1, int v2[], double[] pred) {\n\t\tint v21 = v2[0];\n\t\tint v22 = v2[1];\n\t\tint vMV = v2[2];\n\t\tdouble[] resp1 = table.resp[v1][v21];\n\t\tdouble[] resp2 = table.resp[v1][v22];\n\t\tdouble[] count1 = table.count[v1][v21];\n\t\tdouble[] count2 = table.count[v1][v22];\n\t\t\n\t\tdouble rss = 0;\n\t\trss += pred[0] * pred[0] * count1[0];\n\t\trss += pred[1] * pred[1] * count1[1];\n\t\trss += pred[2] * pred[2] * count2[2];\n\t\trss += pred[3] * pred[3] * count2[3];\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] countOnMV1 = table.countOnMV1[vMV];\n\t\t\trss += pred[4] * pred[4] * countOnMV1[0];\n\t\t\trss += pred[5] * pred[5] * countOnMV1[1];\n\t\t}\n\t\tdouble[] countOnMV2 = table.countOnMV2[v1];\n\t\trss += pred[6] * pred[6] * countOnMV2[0];\n\t\trss += pred[7] * pred[7] * countOnMV2[1];\n\t\trss += pred[8] * pred[8] * table.countOnMV12;\n\n\t\tdouble t = 0;\n\t\tt += pred[0] * resp1[0];\n\t\tt += pred[1] * resp1[1];\n\t\tt += pred[2] * resp2[2];\n\t\tt += pred[3] * resp2[3];\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] respOnMV1 = table.respOnMV1[vMV];\n\t\t\tt += pred[4] * respOnMV1[0];\n\t\t\tt += pred[5] * respOnMV1[1];\n\t\t}\n\t\tdouble[] respOnMV2 = table.respOnMV2[v1];\n\t\tt += pred[6] * respOnMV2[0];\n\t\tt += pred[7] * respOnMV2[1];\n\t\tt += pred[8] * table.respOnMV12;\n\t\trss -= 2 * t;\n\t\treturn rss;\n\t}\n\n\tprotected static double getRSS(Histogram2D.Table table, int[] v1, int v2, double[] pred) {\n\t\tint v11 = v1[0];\n\t\tint v12 = v1[1];\n\t\tint vMV = v1[2];\n\t\tdouble[] resp1 = table.resp[v11][v2];\n\t\tdouble[] resp2 = table.resp[v12][v2];\n\t\tdouble[] count1 = table.count[v11][v2];\n\t\tdouble[] count2 = table.count[v12][v2];\n\t\t\n\t\tdouble rss = 0;\n\t\trss += pred[0] * pred[0] * count1[0];\n\t\trss += pred[1] * pred[1] * count1[2];\n\t\trss += pred[2] * pred[2] * count2[1];\n\t\trss += pred[3] * pred[3] * count2[3];\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] countOnMV2 = table.countOnMV2[vMV];\n\t\t\trss += pred[4] * pred[4] * countOnMV2[0];\n\t\t\trss += pred[5] * pred[5] * countOnMV2[1];\n\t\t}\n\t\tdouble[] countOnMV1 = table.countOnMV1[v2];\n\t\trss += pred[6] * pred[6] * countOnMV1[0];\n\t\trss += pred[7] * pred[7] * countOnMV1[1];\n\t\trss += pred[8] * pred[8] * table.countOnMV12;\n\n\t\tdouble t = 0;\n\t\tt += pred[0] * resp1[0];\n\t\tt += pred[1] * resp1[2];\n\t\tt += pred[2] * resp2[1];\n\t\tt += pred[3] * resp2[3];\n\t\tif (vMV >= 0) {\n\t\t\tdouble[] respOnMV2 = table.respOnMV2[vMV];\n\t\t\tt += pred[4] * respOnMV2[0];\n\t\t\tt += pred[5] * respOnMV2[1];\n\t\t}\n\t\tdouble[] respOnMV1 = table.respOnMV1[v2];\n\t\tt += pred[6] * respOnMV1[0];\n\t\tt += pred[7] * respOnMV1[1];\n\t\tt += pred[8] * table.respOnMV12;\n\t\trss -= 2 * t;\n\n\t\treturn rss;\n\t}\n\n\tprotected static Function2D getFunction2D(int attIndex1, int attIndex2, int v1, int[] v2, double[] predInt) {\n\t\tdouble[] splits1 = new double[] { v1, Double.POSITIVE_INFINITY };\n\t\tdouble[] splits2 = null;\n\t\tdouble[][] predictions = null;\n\t\tdouble[] predictionsOnMV1 = null;\n\t\tdouble[] predictionsOnMV2 = new double[] {predInt[6], predInt[7]};\n\t\tif (v2[0] < v2[1]) {\n\t\t\tif (v2[2] < 0 || v2[2] == v2[0] || v2[2] == v2[1]) {\n\t\t\t\tsplits2 = new double[] { v2[0], v2[1], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[1], predInt[1] },\n\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[3] } \n\t\t\t\t};\n\t\t\t\tif (v2[2] < 0) {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { 0.0, 0.0, 0.0 };\n\t\t\t\t} else if (v2[2] == v2[0]) {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v2[2] < v2[0]) {\n\t\t\t\t\tsplits2 = new double[] { v2[2], v2[0], v2[1], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[1], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[2], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5], predInt[5], predInt[5] };\n\t\t\t\t} else if (v2[2] < v2[1]) {\n\t\t\t\t\tsplits2 = new double[] { v2[0], v2[2], v2[1], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[1], predInt[1], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[2], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits2 = new double[] { v2[0], v2[1], v2[2], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[1], predInt[1], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[3], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t} else if (v2[0] > v2[1]) {\n\t\t\tif (v2[2] < 0 || v2[2] == v2[1] || v2[2] == v2[1]) {\n\t\t\t\tsplits2 = new double[] { v2[1], v2[0], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[1] },\n\t\t\t\t\t\t{ predInt[2], predInt[3], predInt[3] } \n\t\t\t\t};\n\t\t\t\tif (v2[2] < 0) {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { 0.0, 0.0, 0.0 };\n\t\t\t\t} else if (v2[2] == v2[1]) {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v2[2] < v2[1]) {\n\t\t\t\t\tsplits2 = new double[] { v2[2], v2[1], v2[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[0], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[3], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5], predInt[5], predInt[5] };\n\t\t\t\t} else if (v2[2] < v2[0]) {\n\t\t\t\t\tsplits2 = new double[] { v2[1], v2[2], v2[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[0], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[3], predInt[3], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits2 = new double[] { v2[1], v2[0], v2[2], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[1], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[3], predInt[3], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// v2[0] == v2[1]\n\t\t\tif (v2[2] < 0 || v2[2] == v2[0]) {\n\t\t\t\tsplits2 = new double[] { v2[0], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[1] }, \n\t\t\t\t\t\t{ predInt[2], predInt[3] }\n\t\t\t\t};\n\t\t\t\tif (v2[2] < 0) {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { 0.0, 0.0 };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v2[2] < v2[0]) {\n\t\t\t\t\tsplits2 = new double[] { v2[2], v2[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[0], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[2], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits2 = new double[] { v2[0], v2[2], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[1], predInt[1] },\n\t\t\t\t\t\t\t{ predInt[2], predInt[3], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV1 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn new Function2D(attIndex1, attIndex2, splits1, splits2, predictions,\n\t\t\t\tpredictionsOnMV1, predictionsOnMV2, predInt[8]);\n\t}\n\n\tprotected static Function2D getFunction2D(int attIndex1, int attIndex2, int[] v1, int v2, double[] predInt) {\n\t\tdouble[] splits1 = null;\n\t\tdouble[] splits2 = new double[] { v2, Double.POSITIVE_INFINITY };\n\t\tdouble[] predictionsOnMV1 = new double[] {predInt[6], predInt[7]};\n\t\tdouble[] predictionsOnMV2 = null;\n\t\tdouble[][] predictions = null;\n\t\tif (v1[0] < v1[1]) {\n\t\t\tif (v1[2] < 0 || v1[2] == v1[0] || v1[2] == v1[1]) {\n\t\t\t\tsplits1 = new double[] { v1[0], v1[1], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[2] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] }\n\t\t\t\t};\n\t\t\t\tif (v1[2] < 0) {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { 0.0, 0.0, 0.0 };\n\t\t\t\t} else if (v1[2] == v1[0]) {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v1[2] < v1[0]) {\n\t\t\t\t\tsplits1 = new double[] { v1[2], v1[0], v1[1], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[2] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5], predInt[5], predInt[5] };\n\t\t\t\t} else if (v1[2] < v1[1]) {\n\t\t\t\t\tsplits1 = new double[] { v1[0], v1[2], v1[1], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[2] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits1 = new double[] { v1[0], v1[1], v1[2], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[3] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t} else if (v1[0] > v1[1]) {\n\t\t\tif (v1[2] < 0 || v1[2] == v1[0] || v1[2] == v1[1]) {\n\t\t\t\tsplits1 = new double[] { v1[1], v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[3] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t};\n\t\t\t\tif (v1[2] < 0) {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { 0.0, 0.0, 0.0 };\n\t\t\t\t} else if (v1[2] == v1[0]) {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v1[2] < v1[1]) {\n\t\t\t\t\tsplits1 = new double[] { v1[2], v1[1], v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[3] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5], predInt[5], predInt[5] };\n\t\t\t\t} else if (v1[2] < v1[0]) {\n\t\t\t\t\tsplits1 = new double[] { v1[1], v1[2], v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[3] }, \n\t\t\t\t\t\t{ predInt[0], predInt[3] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits1 = new double[] { v1[1], v1[0], v1[2], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[0], predInt[3] },\n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// v1[0] ==  v1[1]\n\t\t\tif (v1[2] < 0 || v1[2] == v1[0]) {\n\t\t\t\tsplits1 = new double[] { v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t};\n\t\t\t\tif (v1[2] < 0) {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { 0.0, 0.0 };\n\t\t\t\t} else {\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif (v1[2] < v1[0]) {\n\t\t\t\t\tsplits1 = new double[] { v1[2], v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[5], predInt[5] };\n\t\t\t\t} else {\n\t\t\t\t\tsplits1 = new double[] { v1[2], v1[0], Double.POSITIVE_INFINITY };\n\t\t\t\t\tpredictions = new double[][] { \n\t\t\t\t\t\t\t{ predInt[0], predInt[2] }, \n\t\t\t\t\t\t\t{ predInt[1], predInt[3] }, \n\t\t\t\t\t\t\t{ predInt[1], predInt[3] } \n\t\t\t\t\t};\n\t\t\t\t\tpredictionsOnMV2 = new double[] { predInt[4], predInt[4], predInt[5] };\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn new Function2D(attIndex1, attIndex2, splits1, splits2, predictions,\n\t\t\t\tpredictionsOnMV1, predictionsOnMV2, predInt[8]);\n\t}\n\n\tprotected static void lineSearch(Instances instances, int attIndex1, int attIndex2, int c1, int c21, int c22, int cMV, \n\t\t\tdouble[] predictions) {\n\t\tdouble[] numerator = new double[9];\n\t\tdouble[] denominator = new double[9];\n\t\tfor (Instance instance : instances) {\n\t\t\tfinal double target = instance.getTarget();\n\t\t\tfinal double weight = instance.getWeight();\n\t\t\tfinal double t = Math.abs(target);\n\t\t\tfinal double num = target * weight;\n\t\t\tfinal double den = t * (1 - t) * weight;\n\t\t\tif (!instance.isMissing(attIndex1) && !instance.isMissing(attIndex2)) {\n\t\t\t\tint v1 = (int) instance.getValue(attIndex1);\n\t\t\t\tint v2 = (int) instance.getValue(attIndex2);\n\t\t\t\tif (v1 <= c1) {\n\t\t\t\t\tif (v2 <= c21) {\n\t\t\t\t\t\tnumerator[0] += num;\n\t\t\t\t\t\tdenominator[0] += den;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tnumerator[1] += num;\n\t\t\t\t\t\tdenominator[1] += den;\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif (v2 <= c22) {\n\t\t\t\t\t\tnumerator[2] += num;\n\t\t\t\t\t\tdenominator[2] += den;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tnumerator[3] += num;\n\t\t\t\t\t\tdenominator[3] += den;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if (instance.isMissing(attIndex1) && !instance.isMissing(attIndex2)) {\n\t\t\t\tint v2 = (int) instance.getValue(attIndex2);\n\t\t\t\tif (cMV >= 0) {\n\t\t\t\t\tif (v2 <= cMV) {\n\t\t\t\t\t\tnumerator[4] += num;\n\t\t\t\t\t\tdenominator[4] += den;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tnumerator[5] += num;\n\t\t\t\t\t\tdenominator[5] += den;\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tthrow new RuntimeException(\"Something went wrong\");\n\t\t\t\t}\n\t\t\t} else if (!instance.isMissing(attIndex1) && instance.isMissing(attIndex2)) {\n\t\t\t\tint v1 = (int) instance.getValue(attIndex1);\n\t\t\t\tif (v1 <= c1) {\n\t\t\t\t\tnumerator[6] += num;\n\t\t\t\t\tdenominator[6] += den;\n\t\t\t\t} else {\n\t\t\t\t\tnumerator[7] += num;\n\t\t\t\t\tdenominator[7] += den;\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tnumerator[8] += num;\n\t\t\t\tdenominator[8] += den;\n\t\t\t}\n\t\t}\n\t\tfor (int i = 0; i < numerator.length; i++) {\n\t\t\tpredictions[i] = MathUtils.divide(numerator[i], denominator[i], 0);\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/SubagSequence.java",
    "content": "package mltk.predictor.function;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.PriorityQueue;\nimport java.util.Set;\n\nimport mltk.util.Element;\nimport mltk.util.Permutation;\nimport mltk.util.Queue;\nimport mltk.util.UFSets;\nimport mltk.util.tuple.IntPair;\n\nclass SubagSequence {\n\t\n\tstatic class SampleDelta {\n\t\t\n\t\tint[] toAdd;\n\t\tint[] toDel;\n\t\t\n\t\tSampleDelta(Set<Integer> toAdd, Set<Integer> toDel) {\n\t\t\tthis.toAdd = new int[toAdd.size()];\n\t\t\tint k = 0;\n\t\t\tfor (Integer idx : toAdd) {\n\t\t\t\tthis.toAdd[k++] = idx;\n\t\t\t}\n\t\t\tk = 0;\n\t\t\tthis.toDel = new int[toDel.size()];\n\t\t\tfor (Integer idx : toDel) {\n\t\t\t\tthis.toDel[k++] = idx;\n\t\t\t}\n\t\t}\n\t\t\n\t\tint getDistance() {\n\t\t\treturn toAdd.length + toDel.length;\n\t\t}\n\t\t\n\t}\n\t\n\tstatic class Sample {\n\t\t\n\t\tSet<Integer> set;\n\t\tint[] indices;\n\t\t\n\t\tSample(Set<Integer> set) {\n\t\t\tthis.set = set;\n\t\t\tthis.indices = new int[set.size()];\n\t\t\tint k = 0;\n\t\t\tfor (Integer idx : set) {\n\t\t\t\tthis.indices[k++] = idx;\n\t\t\t}\n\t\t}\n\t\t\n\t\tstatic int computeDistance(Sample s1, Sample s2) {\n\t\t\tSampleDelta delta = computeDelta(s1, s2);\n\t\t\treturn delta.getDistance();\n\t\t}\n\t\t\n\t\tstatic SampleDelta computeDelta(Sample s1, Sample s2) {\n\t\t\tSet<Integer> toAdd = new HashSet<>(s2.set);\n\t\t\ttoAdd.removeAll(s1.set);\n\t\t\tSet<Integer> toDel = new HashSet<>(s1.set);\n\t\t\ttoDel.removeAll(s2.set);\n\t\t\t\n\t\t\treturn new SampleDelta(toAdd, toDel);\n\t\t}\n\t\t\n\t\tint getWeight() {\n\t\t\treturn set.size();\n\t\t}\n\t\t\n\t}\n\t\n\tSample[] samples;\n\tint[] start;\n\tint[] end;\n\tSampleDelta[] deltas;\n\tint[] count;\n\n\tSubagSequence(int n, int m, int baggingIters) {\n\t\tsamples = new Sample[baggingIters];\n\t\tcount = new int[baggingIters];\n\t\tfor (int i = 0; i < samples.length; i++) {\n\t\t\tsamples[i] = createSubsample(n, m);\n\t\t}\n\t\tcomputeSequence(samples);\n\t}\n\t\n\tSample createSubsample(int n, int m) {\n\t\tPermutation perm = new Permutation(n);\n\t\tperm.permute();\n\t\tint[] a = perm.getPermutation();\n\t\tSet<Integer> set = new HashSet<>(m);\n\t\tfor (int i = 0; i < m; i++) {\n\t\t\tset.add(a[i]);\n\t\t}\n\t\t\n\t\treturn new Sample(set);\n\t}\n\t\n\tprivate void computeSequence(Sample[] samples) {\n\t\tPriorityQueue<Element<IntPair>> q = new PriorityQueue<>(samples.length * (samples.length - 1) / 2);\n\t\tfor (int i = 0; i < samples.length - 1; i++) {\n\t\t\tfor (int j = i + 1; j < samples.length; j++) {\n\t\t\t\tint distance = Sample.computeDistance(samples[i], samples[j]);\n\t\t\t\tq.add(new Element<IntPair>(new IntPair(i, j), distance));\n\t\t\t}\n\t\t}\n\t\t\n\t\tMap<Integer, Set<Integer>> map = new HashMap<>();\n\t\t\n\t\tUFSets ufsets = new UFSets(samples.length);\n\t\twhile (!q.isEmpty()) {\n\t\t\tElement<IntPair> e = q.poll();\n\t\t\tint x = e.element.v1;\n\t\t\tint y = e.element.v2;\n\t\t\tint root1 = ufsets.find(x);\n\t\t\tint root2 = ufsets.find(y);\n\t\t\tif (root1 != root2) {\t\n\t\t\t\tufsets.union(root1, root2);\n\t\t\t\tif (!map.containsKey(x)) {\n\t\t\t\t\tmap.put(x, new HashSet<>());\n\t\t\t\t}\n\t\t\t\tmap.get(x).add(y);\n\t\t\t\tif (!map.containsKey(y)) {\n\t\t\t\t\tmap.put(y, new HashSet<>());\n\t\t\t\t}\n\t\t\t\tmap.get(y).add(x);\n\t\t\t}\n\t\t}\n\t\t\n\t\t\n\t\tint s = 0;\n\t\tfor (int i = 1; i < samples.length; i++) {\n\t\t\tint weight = samples[i].getWeight();\n\t\t\tif (weight < samples[s].getWeight()) {\n\t\t\t\ts = i;\n\t\t\t}\n\t\t}\n\t\t\n\t\tList<Integer> fromList = new ArrayList<>(samples.length);\n\t\tList<Integer> toList = new ArrayList<>(samples.length);\n\t\tList<SampleDelta> deltas = new ArrayList<>(samples.length);\n\t\t\n\t\t// BFS\n\t\tSet<Integer> covered = new HashSet<>();\n\t\tcovered.add(s);\n\t\tQueue<Integer> queue = new Queue<>();\n\t\tqueue.enqueue(s);\n\t\twhile (covered.size() < samples.length) {\n\t\t\tInteger node = queue.dequeue();\n\t\t\tSet<Integer> children = map.get(node);\n\t\t\tfor (Integer child : children) {\n\t\t\t\tif (covered.contains(child)) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tcovered.add(child);\n\t\t\t\tfromList.add(node);\n\t\t\t\ttoList.add(child);\n\t\t\t\tSampleDelta delta = Sample.computeDelta(samples[node], samples[child]);\n\t\t\t\tdeltas.add(delta);\n\t\t\t\tqueue.enqueue(child);\n\t\t\t}\n\t\t}\n\t\t\n\t\tthis.start = new int[fromList.size()];\n\t\tthis.end = new int[toList.size()];\n\t\tthis.deltas = new SampleDelta[deltas.size()];\n\t\tfor (int i = 0; i < this.start.length; i++) {\n\t\t\tthis.start[i] = fromList.get(i);\n\t\t\tthis.end[i] = toList.get(i);\n\t\t\tthis.deltas[i] = deltas.get(i);\n\t\t}\n\t\tfor (int i = 0; i < this.start.length; i++) {\n\t\t\tcount[this.start[i]]++;\n\t\t}\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/SubaggedLineCutter.java",
    "content": "package mltk.predictor.function;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.predictor.function.SubagSequence.SampleDelta;\nimport mltk.util.MathUtils;\n\n/**\n * Class for cutting lines with subagging.\n * \n * @author Yin Lou\n *\n */\npublic class SubaggedLineCutter extends EnsembledLineCutter {\n\t\n\tprivate int subsampleSize;\n\t\n\tprivate SubagSequence ss;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic SubaggedLineCutter() {\n\t\tthis(false);\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param isClassification {@code true} if it is a classification problem.\n\t */\n\tpublic SubaggedLineCutter(boolean isClassification) {\n\t\tattIndex = -1;\n\t\tthis.isClassification = isClassification;\n\t}\n\t\n\t/**\n\t * Creates internal subsamples.\n\t * \n\t * @param n the size of the data set.\n\t * @param subsampleRatio the subsample ratio.\n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void createSubags(int n, double subsampleRatio, int baggingIters) {\n\t\tthis.subsampleSize = (int) (n * subsampleRatio);\n\t\tss = new SubagSequence(n, subsampleSize, baggingIters);\n\t}\n\n\t@Override\n\tpublic BaggedEnsemble build(Instances instances) {\n\t\treturn build(instances, attIndex, numIntervals);\n\t}\n\t\n\tclass Histogram {\n\t\tdouble[][] histogram;\n\t\t\n\t\tHistogram(double[][] histogram) {\n\t\t\tthis.histogram = histogram;\n\t\t}\n\t\t\n\t\tHistogram copy() {\n\t\t\tdouble[][] newHistogram = new double[histogram.length][histogram[0].length];\n\t\t\tfor (int i = 0; i < histogram.length; i++) {\n\t\t\t\tdouble[] hist = histogram[i];\n\t\t\t\tdouble[] newHist = newHistogram[i];\n\t\t\t\tfor (int j = 0; j < hist.length; j++) {\n\t\t\t\t\tnewHist[j] = hist[j];\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn new Histogram(newHistogram);\n\t\t}\n\t}\n\t\n\t/**\n\t * Builds an 1D function ensemble.\n\t * \n\t * @param instances the training set.\n\t * @param attIndex the attribute index.\n\t * @param numIntervals the number of intervals.\n\t * @return an 1D function ensemble.\n\t */\n\tpublic BaggedEnsemble build(Instances instances, int attIndex, int numIntervals) {\n\t\tAttribute attribute = instances.getAttributes().get(attIndex);\n\t\treturn build(instances, attribute, numIntervals);\n\t}\n\t\n\t\n\t/**\n\t * Builds an 1D function ensemble.\n\t * \n\t * @param instances the training set.\n\t * @param attribute the attribute.\n\t * @param numIntervals the number of intervals.\n\t * @return an 1D function ensemble.\n\t */\n\tpublic BaggedEnsemble build(Instances instances, Attribute attribute, int numIntervals) {\n\t\tif (ss == null) {\n\t\t\tss = new SubagSequence(instances.size(), subsampleSize, baggingIters);\n\t\t}\n\t\t\n\t\tSubagSequence.Sample[] samples = ss.samples;\n\t\tint[] start = ss.start;\n\t\tint[] end = ss.end;\n\t\tSampleDelta[] deltas = ss.deltas;\n\t\tint[] count = Arrays.copyOf(ss.count, ss.count.length);\n\t\t\n\t\tFunction1D[] funcs = new Function1D[ss.samples.length];\n\t\t\n\t\tint attIndex = attribute.getIndex();\n\t\t\n\t\tdouble[] targets = new double[instances.size()];\n\t\tdouble[] fvalues = new double[instances.size()];\n\t\tdouble[] weights = new double[instances.size()];\n\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\tInstance instance = instances.get(i);\n\t\t\ttargets[i] = instance.getTarget();\n\t\t\tfvalues[i] = instance.getValue(attribute);\n\t\t\tweights[i] = instance.getWeight();\n\t\t}\n\n\t\tif (attribute.getType() == Attribute.Type.NUMERIC) {\n\t\t\tthrow new RuntimeException(\"Not implemented yet!\");\n\t\t} else {\n\t\t\tint size = 0;\n\t\t\tif (attribute.getType() == Attribute.Type.BINNED) {\n\t\t\t\tsize = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t} else {\n\t\t\t\tsize = ((NominalAttribute) attribute).getCardinality();\n\t\t\t}\n\t\t\t\n\t\t\tHistogram[] histograms = new Histogram[ss.samples.length];\n\t\t\t{// Initialization\n\t\t\t\tdouble[][] histogram = new double[size + 1][2];\n\t\t\t\tfor (int index : samples[start[0]].indices) {\n\t\t\t\t\tdouble value = fvalues[index];\n\t\t\t\t\tdouble weight = weights[index];\n\t\t\t\t\tdouble target = targets[index];\n\t\t\t\t\tint idx = histogram.length - 1;\n\t\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\t\tidx = (int) value;\n\t\t\t\t\t}\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\thistogram[idx][0] += target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\thistogram[idx][0] += target * weight;\n\t\t\t\t\t}\n\t\t\t\t\thistogram[idx][1] += weight;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\thistograms[0] = new Histogram(histogram);\n\t\t\t\tfuncs[0] = buildFromHistogram(attIndex, histograms[0]);\n\t\t\t}\n\t\t\t\n\t\t\tfor (int k = 0; k < start.length; k++) {\n\t\t\t\tint from = start[k];\n\t\t\t\tHistogram histStart = histograms[from];\n\t\t\t\tcount[from]--;\n\t\t\t\t\n\t\t\t\t// Generate histogram for its child\n\t\t\t\tint to = end[k];\n\t\t\t\tHistogram histEnd = null;\n\t\t\t\tif (count[from] == 0) {\n\t\t\t\t\thistEnd = histStart;\n\t\t\t\t} else {\n\t\t\t\t\thistEnd = histStart.copy();\n\t\t\t\t}\n\t\t\t\tSampleDelta delta = deltas[k];\n\t\t\t\tfor (int index : delta.toAdd) {\n\t\t\t\t\tdouble value = fvalues[index];\n\t\t\t\t\tdouble weight = weights[index];\n\t\t\t\t\tdouble target = targets[index];\n\t\t\t\t\tint idx = histEnd.histogram.length - 1;\n\t\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\t\tidx = (int) value;\n\t\t\t\t\t}\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\thistEnd.histogram[idx][0] += target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\thistEnd.histogram[idx][0] += target * weight;\n\t\t\t\t\t}\n\t\t\t\t\thistEnd.histogram[idx][1] += weight;\n\t\t\t\t}\n\t\t\t\tfor (int index : delta.toDel) {\n\t\t\t\t\tdouble value = fvalues[index];\n\t\t\t\t\tdouble weight = weights[index];\n\t\t\t\t\tdouble target = targets[index];\n\t\t\t\t\tint idx = histEnd.histogram.length - 1;\n\t\t\t\t\tif (!Double.isNaN(value)) {\n\t\t\t\t\t\tidx = (int) value;\n\t\t\t\t\t}\n\t\t\t\t\tif (isClassification) {\n\t\t\t\t\t\thistEnd.histogram[idx][0] -= target;\n\t\t\t\t\t} else {\n\t\t\t\t\t\thistEnd.histogram[idx][0] -= target * weight;\n\t\t\t\t\t}\n\t\t\t\t\thistEnd.histogram[idx][1] -= weight;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\thistograms[to] = histEnd;\n\t\t\t\tfuncs[to] = buildFromHistogram(attIndex, histEnd);\n\t\t\t}\n\t\t}\n\t\t\n\t\tBaggedEnsemble ensemble = new BaggedEnsemble(ss.samples.length);\n\t\tfor (Function1D func : funcs) {\n\t\t\tensemble.add(func);\n\t\t}\n\t\t\n\t\treturn ensemble;\n\t}\n\t\n\tprotected Function1D buildFromHistogram(int attIndex, Histogram histogram) {\n\t\tfinal int size = histogram.histogram.length;\n\t\tList<double[]> histograms = new ArrayList<>(size);\n\t\tfor (int i = 0; i < size - 1; i++) {\n\t\t\tif (!MathUtils.isZero(histogram.histogram[i][1])) {\n\t\t\t\tdouble[] hist = histogram.histogram[i];\n\t\t\t\thistograms.add(new double[] {i, hist[0], hist[1]});\n\t\t\t}\n\t\t}\n\t\thistograms.add(new double[] { Double.NaN, histogram.histogram[size - 1][0], histogram.histogram[size - 1][1] });\n\t\t\n\t\tFunction1D func = LineCutter.build(attIndex, histograms, numIntervals);\n\t\treturn func;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/UnivariateFunction.java",
    "content": "package mltk.predictor.function;\n\n/**\n * Interface for univariate functions.\n * \n * @author Yin Lou\n * \n */\npublic interface UnivariateFunction {\n\n\t/**\n\t * Computes the value for the function.\n\t * \n\t * @param x the argument.\n\t * @return the value for the function.\n\t */\n\tpublic double evaluate(double x);\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/function/package-info.java",
    "content": "/**\n * Provides classes for simple functions, such as univariate/bivariate functions.\n */\npackage mltk.predictor.function;"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/DenseDesignMatrix.java",
    "content": "package mltk.predictor.gam;\r\n\r\nimport java.util.HashSet;\r\nimport java.util.Set;\r\n\r\nimport mltk.predictor.function.CubicSpline;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.VectorUtils;\r\n\r\nclass DenseDesignMatrix {\r\n\r\n\tdouble[][][] x;\r\n\tdouble[][] knots;\r\n\tdouble[][] std;\r\n\r\n\tDenseDesignMatrix(double[][][] x, double[][] knots, double[][] std) {\r\n\t\tthis.x = x;\r\n\t\tthis.knots = knots;\r\n\t\tthis.std = std;\r\n\t}\r\n\r\n\tstatic DenseDesignMatrix createCubicSplineDesignMatrix(double[][] dataset, double[] stdList, int numKnots) {\r\n\t\tfinal int n = dataset[0].length;\r\n\t\tfinal int p = dataset.length;\r\n\t\tdouble[][][] x = new double[p][][];\r\n\t\tdouble[][] knots = new double[p][];\r\n\t\tdouble[][] std = new double[p][];\r\n\t\tdouble factor = Math.sqrt(n);\r\n\t\tfor (int j = 0; j < dataset.length; j++) {\r\n\t\t\tSet<Double> uniqueValues = new HashSet<>();\r\n\t\t\tdouble[] x1 = dataset[j];\r\n\t\t\tfor (int i = 0; i < n; i++) {\r\n\t\t\t\tuniqueValues.add(x1[i]);\r\n\t\t\t}\r\n\t\t\tint nKnots = uniqueValues.size() <= numKnots ? 0 : numKnots;\r\n\t\t\tknots[j] = new double[nKnots];\r\n\t\t\tif (nKnots != 0) {\r\n\t\t\t\tx[j] = new double[nKnots + 3][];\r\n\t\t\t\tstd[j] = new double[nKnots + 3];\r\n\t\t\t} else {\r\n\t\t\t\tx[j] = new double[1][];\r\n\t\t\t\tstd[j] = new double[1];\r\n\t\t\t}\r\n\t\t\tdouble[][] t = x[j];\r\n\t\t\tt[0] = x1;\r\n\t\t\tstd[j][0] = stdList[j] / factor;\r\n\t\t\tif (nKnots != 0) {\r\n\t\t\t\tdouble[] x2 = new double[n];\r\n\t\t\t\tfor (int i = 0; i < n; i++) {\r\n\t\t\t\t\tx2[i] = x1[i] * x1[i];\r\n\t\t\t\t}\r\n\t\t\t\tt[1] = x2;\r\n\t\t\t\tdouble[] x3 = new double[n];\r\n\t\t\t\tfor (int i = 0; i < n; i++) {\r\n\t\t\t\t\tx3[i] = x2[i] * x1[i];\r\n\t\t\t\t}\r\n\t\t\t\tt[2] = x3;\r\n\r\n\t\t\t\tstd[j][1] = StatUtils.sd(x2) / factor;\r\n\t\t\t\tstd[j][2] = StatUtils.sd(x3) / factor;\r\n\r\n\t\t\t\tdouble max = StatUtils.max(x1);\r\n\t\t\t\tdouble min = StatUtils.min(x1);\r\n\t\t\t\tdouble stepSize = (max - min) / nKnots;\r\n\t\t\t\tfor (int k = 0; k < nKnots; k++) {\r\n\t\t\t\t\tknots[j][k] = min + stepSize * k;\r\n\t\t\t\t\tdouble[] basis = new double[n];\r\n\t\t\t\t\tfor (int i = 0; i < n; i++) {\r\n\t\t\t\t\t\tbasis[i] = CubicSpline.h(x1[i], knots[j][k]);\r\n\t\t\t\t\t}\r\n\t\t\t\t\tstd[j][k + 3] = StatUtils.sd(basis) / factor;\r\n\t\t\t\t\tt[k + 3] = basis;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\t\r\n\t\t// Normalize the inputs\r\n\t\tfor (int j = 0; j < p; j++) {\r\n\t\t\tdouble[][] block = x[j];\r\n\t\t\tdouble[] s = std[j];\r\n\t\t\tfor (int i = 0; i < block.length; i++) {\r\n\t\t\t\tVectorUtils.divide(block[i], s[i]);\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn new DenseDesignMatrix(x, knots, std);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/GA2MLearner.java",
    "content": "package mltk.predictor.gam;\n\nimport java.io.BufferedReader;\nimport java.io.FileReader;\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.HoldoutValidatedLearnerWithTaskOptions;\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.Sampling;\nimport mltk.core.Attribute.Type;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.predictor.BaggedEnsembleLearner;\nimport mltk.predictor.BoostedEnsemble;\nimport mltk.predictor.HoldoutValidatedLearner;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.function.Array2D;\nimport mltk.predictor.function.CompressionUtils;\nimport mltk.predictor.function.Function2D;\nimport mltk.predictor.function.SquareCutter;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.OptimUtils;\nimport mltk.util.Random;\nimport mltk.util.tuple.IntPair;\n\n/**\n * Class for learning GA^2M models via gradient boosting.\n * \n * <p>\n * Reference:<br>\n * Y. Lou, R. Caruana, J. Gehrke, and G. Hooker. Accurate intelligible models with pairwise interactions. In\n * <i>Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD)</i>,\n * Chicago, IL, USA, 2013.\n * </p>\n * \n * @author Yin Lou\n * \n */\npublic class GA2MLearner extends HoldoutValidatedLearner {\n\t\n\tstatic class Options extends HoldoutValidatedLearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-i\", description = \"input model path\", required = true)\n\t\tString inputModelPath = null;\n\n\t\t@Argument(name = \"-I\", description = \"list of pairwise interactions path\", required = true)\n\t\tString interactionsPath = null;\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-b\", description = \"bagging iterations (default: 100)\")\n\t\tint baggingIters = 100;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t\t@Argument(name = \"-l\", description = \"learning rate (default: 0.01)\")\n\t\tdouble learningRate = 0.01;\n\n\t}\n\n\t/**\n\t * Trains a GA2M.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.GA2MLearner\n\t * -t\ttrain set path\n\t * -i\tinput model path\n\t * -I\tlist of pairwise interactions path\n\t * -m\tmaximum number of iterations\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-v]\tvalid set path\n\t * [-e]\tevaluation metric (default: default metric of task)\n\t * [-S]\tconvergence criteria (default: -1)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-b]\tbagging iterations (default: 100)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * [-l]\tlearning rate (default: 0.01)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(GA2MLearner.class, opts);\n\t\tTask task = null;\n\t\tMetric metric = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t\tif (opts.metric == null) {\n\t\t\t\tmetric = task.getDefaultMetric();\n\t\t\t} else {\n\t\t\t\tmetric = MetricFactory.getMetric(opts.metric);\n\t\t\t}\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\t\t\n\t\tConvergenceTester ct = ConvergenceTester.parse(opts.cc);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tList<IntPair> terms = new ArrayList<>();\n\t\tBufferedReader br = new BufferedReader(new FileReader(opts.interactionsPath));\n\t\tfor (;;) {\n\t\t\tString line = br.readLine();\n\t\t\tif (line == null) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tString[] data = line.split(\"\\\\s+\");\n\t\t\tIntPair term = new IntPair(Integer.parseInt(data[0]), Integer.parseInt(data[1]));\n\t\t\tterms.add(term);\n\t\t}\n\t\tbr.close();\n\n\t\tGAM gam = PredictorReader.read(opts.inputModelPath, GAM.class);\n\n\t\tGA2MLearner learner = new GA2MLearner();\n\t\tlearner.setBaggingIters(opts.baggingIters);\n\t\tlearner.setGAM(gam);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setTask(task);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setConvergenceTester(ct);\n\t\tlearner.setPairs(terms);\n\t\tlearner.setLearningRate(opts.learningRate);\n\t\tlearner.setVerbose(opts.verbose);\n\n\t\tif (opts.validPath != null) {\n\t\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\t\t\tlearner.setValidSet(validSet);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tlearner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(gam, opts.outputModelPath);\n\t\t}\n\t}\n\n\tprivate int baggingIters;\n\tprivate int maxNumIters;\n\tprivate Task task;\n\tprivate double learningRate;\n\tprivate GAM gam;\n\tprivate List<IntPair> pairs;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic GA2MLearner() {\n\t\tverbose = false;\n\t\tbaggingIters = 100;\n\t\tmaxNumIters = -1;\n\t\tlearningRate = 0.01;\n\t\ttask = Task.REGRESSION;\n\t\tmetric = task.getDefaultMetric();\n\t}\n\n\t/**\n\t * Returns the number of bagging iterations.\n\t * \n\t * @return the number of bagging iterations.\n\t */\n\tpublic int getBaggingIters() {\n\t\treturn baggingIters;\n\t}\n\n\t/**\n\t * Sets the number of bagging iterations.\n\t * \n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void setBaggingIters(int baggingIters) {\n\t\tthis.baggingIters = baggingIters;\n\t}\n\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\n\t/**\n\t * Returns the learning rate.\n\t * \n\t * @return the learning rate.\n\t */\n\tpublic double getLearningRate() {\n\t\treturn learningRate;\n\t}\n\n\t/**\n\t * Sets the learning rate.\n\t * \n\t * @param learningRate the learning rate.\n\t */\n\tpublic void setLearningRate(double learningRate) {\n\t\tthis.learningRate = learningRate;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the new task.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\n\t/**\n\t * Returns the GAM.\n\t * \n\t * @return the GAM.\n\t */\n\tpublic GAM getGAM() {\n\t\treturn gam;\n\t}\n\n\t/**\n\t * Sets the GAM.\n\t * \n\t * @param gam the GAM.\n\t */\n\tpublic void setGAM(GAM gam) {\n\t\tthis.gam = gam;\n\t}\n\n\t/**\n\t * Returns the list of feature interaction pairs.\n\t * \n\t * @return the list of feature interaction pairs.\n\t */\n\tpublic List<IntPair> getPairs() {\n\t\treturn pairs;\n\t}\n\n\t/**\n\t * Sets the list of feature interaction pairs.\n\t * \n\t * @param pairs the list of feature interaction pairs.\n\t */\n\tpublic void setPairs(List<IntPair> pairs) {\n\t\tthis.pairs = pairs;\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param gam the GAM.\n\t * @param terms the list of feature interaction pairs.\n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void buildClassifier(GAM gam, List<IntPair> terms, Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>(terms.size());\n\t\tint[] indices = new int[terms.size()];\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tindices[i] = indexOf(gam.terms, terms.get(i));\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tInstances[] bags = Sampling.createBags(trainSet, baggingIters);\n\t\tSquareCutter cutter = new SquareCutter(true);\n\t\tBaggedEnsembleLearner learner = new BaggedEnsembleLearner(bags.length, cutter);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tdouble[] pValid = new double[validSet.size()];\n\n\t\tfor (int i = 0; i < pTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\tpTrain[i] = gam.regress(instance);\n\t\t}\n\t\tOptimUtils.computePseudoResidual(pTrain, target, rTrain);\n\t\t\n\t\tfor (int i = 0; i < pValid.length; i++) {\n\t\t\tInstance instance = validSet.get(i);\n\t\t\tpValid[i] = gam.regress(instance);\n\t\t}\n\n\t\t// Gradient boosting\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < terms.size(); j++) {\n\t\t\t\t// Derivitive to attribute k\n\t\t\t\t// Minimizes the loss function: log(1 + exp(-yF))\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\n\t\t\t\t// Train model\n\t\t\t\tIntPair term = terms.get(j);\n\t\t\t\tcutter.setAttIndices(term.v1, term.v2);\n\t\t\t\tBaggedEnsemble baggedEnsemble = learner.build(bags);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\t\t\t\tFunction2D func = (Function2D) baggedEnsemble.get(i);\n\t\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(baggedEnsemble);\n\n\t\t\t\t// Update predictions\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\tpTrain[i] += pred;\n\t\t\t\t\trTrain[i] = OptimUtils.getPseudoResidual(pTrain[i], target[i]);\n\t\t\t\t}\n\t\t\t\tfor (int i = 0; i < validSet.size(); i++) {\n\t\t\t\t\tInstance instance = validSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\tpValid[i] += pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\t\tct.add(measure);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" term \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\n\t\t// Remove trees\n\t\tint n = idx / terms.size();\n\t\tint m = idx % terms.size();\n\t\tfor (int k = 0; k < terms.size(); k++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(k);\n\t\t\tfor (int i = boostedEnsemble.size(); i > n + 1; i--) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t\tif (k > m) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tIntPair term = terms.get(i);\n\t\t\tAttribute f1 = attributes.get(term.v1);\n\t\t\tAttribute f2 = attributes.get(term.v2);\n\t\t\tint n1 = -1;\n\t\t\tif (f1.getType() == Type.BINNED) {\n\t\t\t\tn1 = ((BinnedAttribute) f1).getNumBins();\n\t\t\t} else if (f1.getType() == Type.NOMINAL) {\n\t\t\t\tn1 = ((NominalAttribute) f1).getCardinality();\n\t\t\t}\n\t\t\tint n2 = -1;\n\t\t\tif (f2.getType() == Type.BINNED) {\n\t\t\t\tn2 = ((BinnedAttribute) f2).getNumBins();\n\t\t\t} else if (f2.getType() == Type.NOMINAL) {\n\t\t\t\tn2 = ((NominalAttribute) f2).getCardinality();\n\t\t\t}\n\t\t\tArray2D newRegressor = CompressionUtils.compress(term.v1, term.v2, n1, n2, boostedEnsemble);\n\t\t\tif (indices[i] < 0) {\n\t\t\t\tgam.add(new int[] { term.v1, term.v2 }, newRegressor);\n\t\t\t} else {\n\t\t\t\tRegressor regressor = gam.regressors.get(indices[i]);\n\t\t\t\tif (regressor instanceof Array2D) {\n\t\t\t\t\tArray2D ary = (Array2D) regressor;\n\t\t\t\t\tary.add(newRegressor);\n\t\t\t\t} else {\n\t\t\t\t\tthrow new RuntimeException(\"Failed to add new regressor\");\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param gam the GAM.\n\t * @param terms the list of feature interaction pairs.\n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void buildClassifier(GAM gam, List<IntPair> terms, Instances trainSet, int maxNumIters) {\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>();\n\t\tint[] indices = new int[terms.size()];\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tindices[i] = indexOf(gam.terms, terms.get(i));\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tInstances[] bags = Sampling.createBags(trainSet, baggingIters);\n\t\tSquareCutter cutter = new SquareCutter(true);\n\t\tBaggedEnsembleLearner learner = new BaggedEnsembleLearner(bags.length, cutter);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\n\t\tfor (int i = 0; i < pTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\tpTrain[i] = gam.regress(instance);\n\t\t}\n\t\tOptimUtils.computePseudoResidual(pTrain, target, rTrain);\n\n\t\t// Gradient boosting\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < terms.size(); j++) {\n\t\t\t\t// Derivitive to attribute k\n\t\t\t\t// Minimizes the loss function: log(1 + exp(-yF))\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\n\t\t\t\t// Train model\n\t\t\t\tIntPair term = terms.get(j);\n\t\t\t\tcutter.setAttIndices(term.v1, term.v2);\n\t\t\t\tBaggedEnsemble baggedEnsemble = learner.build(bags);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\t\t\t\tFunction2D func = (Function2D) baggedEnsemble.get(i);\n\t\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(baggedEnsemble);\n\n\t\t\t\t// Update predictions\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\tpTrain[i] += pred;\n\t\t\t\t\trTrain[i] = OptimUtils.getPseudoResidual(pTrain[i], target[i]);\n\t\t\t\t}\n\n\t\t\t\tdouble measure = simpleMetric.eval(pTrain, target);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" term \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tIntPair term = terms.get(i);\n\t\t\tAttribute f1 = attributes.get(term.v1);\n\t\t\tAttribute f2 = attributes.get(term.v2);\n\t\t\tint n1 = -1;\n\t\t\tif (f1.getType() == Type.BINNED) {\n\t\t\t\tn1 = ((BinnedAttribute) f1).getNumBins();\n\t\t\t} else if (f1.getType() == Type.NOMINAL) {\n\t\t\t\tn1 = ((NominalAttribute) f1).getCardinality();\n\t\t\t}\n\t\t\tint n2 = -1;\n\t\t\tif (f2.getType() == Type.BINNED) {\n\t\t\t\tn2 = ((BinnedAttribute) f2).getNumBins();\n\t\t\t} else if (f2.getType() == Type.NOMINAL) {\n\t\t\t\tn2 = ((NominalAttribute) f2).getCardinality();\n\t\t\t}\n\t\t\tArray2D newRegressor = CompressionUtils.compress(term.v1, term.v2, n1, n2, boostedEnsemble);\n\t\t\tif (indices[i] < 0) {\n\t\t\t\tgam.add(new int[] { term.v1, term.v2 }, newRegressor);\n\t\t\t} else {\n\t\t\t\tRegressor regressor = gam.regressors.get(indices[i]);\n\t\t\t\tif (regressor instanceof Array2D) {\n\t\t\t\t\tArray2D ary = (Array2D) regressor;\n\t\t\t\t\tary.add(newRegressor);\n\t\t\t\t} else {\n\t\t\t\t\tthrow new RuntimeException(\"Failed to add new regressor\");\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param gam the GAM.\n\t * @param terms the list of feature interaction pairs.\n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void buildRegressor(GAM gam, List<IntPair> terms, Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>();\n\t\tint[] indices = new int[terms.size()];\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tindices[i] = indexOf(gam.terms, terms.get(i));\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tInstances[] bags = Sampling.createBags(trainSet, baggingIters);\n\n\t\tSquareCutter cutter = new SquareCutter();\n\t\tBaggedEnsembleLearner learner = new BaggedEnsembleLearner(baggingIters, cutter);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tdouble[] pValid = new double[validSet.size()];\n\t\tdouble[] rValid = new double[validSet.size()];\n\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\trTrain[i] = instance.getTarget() - gam.regress(instance);\n\t\t}\n\t\tfor (int i = 0; i < validSet.size(); i++) {\n\t\t\tInstance instance = validSet.get(i);\n\t\t\tpValid[i] = gam.regress(instance);\n\t\t\trValid[i] = instance.getTarget() - pValid[i];\n\t\t}\n\n\t\t// Gradient boosting\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < terms.size(); j++) {\n\t\t\t\t// Derivative to attribute k\n\t\t\t\t// Equivalent to residual\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\t\t\t\t// Prepare training set\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\t\t\t\t// Train model\n\t\t\t\tIntPair term = terms.get(j);\n\t\t\t\tcutter.setAttIndices(term.v1, term.v2);\n\t\t\t\tBaggedEnsemble baggedEnsemble = learner.build(bags);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\t\t\t\tFunction2D func = (Function2D) baggedEnsemble.get(i);\n\t\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(baggedEnsemble);\n\n\t\t\t\t// Update residuals\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\trTrain[i] -= pred;\n\t\t\t\t}\n\t\t\t\tfor (int i = 0; i < rValid.length; i++) {\n\t\t\t\t\tInstance instance = validSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\tpValid[i] += pred;\n\t\t\t\t\trValid[i] -= pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\t\tct.add(measure);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" term \" + j + \":\" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\n\t\t// Remove trees\n\t\tint n = idx / terms.size();\n\t\tint m = idx % terms.size();\n\t\tfor (int k = 0; k < terms.size(); k++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(k);\n\t\t\tfor (int i = boostedEnsemble.size(); i > n + 1; i--) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t\tif (k > m) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tIntPair term = terms.get(i);\n\t\t\tAttribute f1 = attributes.get(term.v1);\n\t\t\tAttribute f2 = attributes.get(term.v2);\n\t\t\tint n1 = -1;\n\t\t\tif (f1.getType() == Type.BINNED) {\n\t\t\t\tn1 = ((BinnedAttribute) f1).getNumBins();\n\t\t\t} else if (f1.getType() == Type.NOMINAL) {\n\t\t\t\tn1 = ((NominalAttribute) f1).getCardinality();\n\t\t\t}\n\t\t\tint n2 = -1;\n\t\t\tif (f2.getType() == Type.BINNED) {\n\t\t\t\tn2 = ((BinnedAttribute) f2).getNumBins();\n\t\t\t} else if (f2.getType() == Type.NOMINAL) {\n\t\t\t\tn2 = ((NominalAttribute) f2).getCardinality();\n\t\t\t}\n\t\t\tArray2D newRegressor = CompressionUtils.compress(term.v1, term.v2, n1, n2, boostedEnsemble);\n\t\t\tif (indices[i] < 0) {\n\t\t\t\tgam.add(new int[] { term.v1, term.v2 }, newRegressor);\n\t\t\t} else {\n\t\t\t\tRegressor regressor = gam.regressors.get(indices[i]);\n\t\t\t\tif (regressor instanceof Array2D) {\n\t\t\t\t\tArray2D ary = (Array2D) regressor;\n\t\t\t\t\tary.add(newRegressor);\n\t\t\t\t} else {\n\t\t\t\t\tthrow new RuntimeException(\"Failed to add new regressor\");\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param gam the GAM.\n\t * @param terms the list of feature interaction pairs.\n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void buildRegressor(GAM gam, List<IntPair> terms, Instances trainSet, int maxNumIters) {\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>();\n\t\tint[] indices = new int[terms.size()];\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tindices[i] = indexOf(gam.terms, terms.get(i));\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tInstances[] bags = Sampling.createBags(trainSet, baggingIters);\n\n\t\tSquareCutter cutter = new SquareCutter();\n\t\tBaggedEnsembleLearner learner = new BaggedEnsembleLearner(baggingIters, cutter);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\tpTrain[i] = gam.regress(instance);\n\t\t\trTrain[i] = instance.getTarget() - pTrain[i];\n\t\t}\n\n\t\t// Gradient boosting\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < terms.size(); j++) {\n\t\t\t\t// Derivative to attribute k\n\t\t\t\t// Equivalent to residual\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\t\t\t\t// Prepare training set\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\t\t\t\t// Train model\n\t\t\t\tIntPair term = terms.get(j);\n\t\t\t\tcutter.setAttIndices(term.v1, term.v2);\n\t\t\t\tBaggedEnsemble baggedEnsemble = learner.build(bags);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfor (int i = 0; i < baggedEnsemble.size(); i++) {\n\t\t\t\t\t\tFunction2D func = (Function2D) baggedEnsemble.get(i);\n\t\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(baggedEnsemble);\n\n\t\t\t\t// Update residuals\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = baggedEnsemble.regress(instance);\n\t\t\t\t\trTrain[i] -= pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = simpleMetric.eval(pTrain, target);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" term \" + j + \":\" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tIntPair term = terms.get(i);\n\t\t\tAttribute f1 = attributes.get(term.v1);\n\t\t\tAttribute f2 = attributes.get(term.v2);\n\t\t\tint n1 = -1;\n\t\t\tif (f1.getType() == Type.BINNED) {\n\t\t\t\tn1 = ((BinnedAttribute) f1).getNumBins();\n\t\t\t} else if (f1.getType() == Type.NOMINAL) {\n\t\t\t\tn1 = ((NominalAttribute) f1).getCardinality();\n\t\t\t}\n\t\t\tint n2 = -1;\n\t\t\tif (f2.getType() == Type.BINNED) {\n\t\t\t\tn2 = ((BinnedAttribute) f2).getNumBins();\n\t\t\t} else if (f2.getType() == Type.NOMINAL) {\n\t\t\t\tn2 = ((NominalAttribute) f2).getCardinality();\n\t\t\t}\n\t\t\tArray2D newRegressor = CompressionUtils.compress(term.v1, term.v2, n1, n2, boostedEnsemble);\n\t\t\tif (indices[i] < 0) {\n\t\t\t\tgam.add(new int[] { term.v1, term.v2 }, newRegressor);\n\t\t\t} else {\n\t\t\t\tRegressor regressor = gam.regressors.get(indices[i]);\n\t\t\t\tif (regressor instanceof Array2D) {\n\t\t\t\t\tArray2D ary = (Array2D) regressor;\n\t\t\t\t\tary.add(newRegressor);\n\t\t\t\t} else {\n\t\t\t\t\tthrow new RuntimeException(\"Failed to add new regressor\");\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t@Override\n\tpublic GAM build(Instances instances) {\n\t\tif (pairs == null) {\n\t\t\tint p = instances.dimension();\n\t\t\tpairs = new ArrayList<IntPair>();\n\t\t\tfor (int i = 0; i < p; i++) {\n\t\t\t\tfor (int j = i + 1; j < p; j++) {\n\t\t\t\t\tpairs.add(new IntPair(i, j));\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tif (validSet != null) {\n\t\t\t\t\tbuildRegressor(gam, pairs, instances, validSet, maxNumIters);\n\t\t\t\t} else {\n\t\t\t\t\tbuildRegressor(gam, pairs, instances, maxNumIters);\n\t\t\t\t}\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tif (validSet != null) {\n\t\t\t\t\tbuildClassifier(gam, pairs, instances, validSet, maxNumIters);\n\t\t\t\t} else {\n\t\t\t\t\tbuildClassifier(gam, pairs, instances, maxNumIters);\n\t\t\t\t}\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn gam;\n\t}\n\t\n\tprivate int indexOf(List<int[]> terms, IntPair pair) {\n\t\tfor (int i = 0; i < terms.size(); i++) {\n\t\t\tint[] term = terms.get(i);\n\t\t\tif (term.length == 2 && term[0] == pair.v1 && term[1] == pair.v2) {\n\t\t\t\treturn i;\n\t\t\t}\n\t\t}\n\t\treturn -1;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/GAM.java",
    "content": "package mltk.predictor.gam;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Iterator;\nimport java.util.List;\n\nimport mltk.core.Instance;\nimport mltk.predictor.ProbabilisticClassifier;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\nimport mltk.util.MathUtils;\n\n/**\n * Class for generalized additive models (GAMs).\n * \n * @author Yin Lou\n * \n */\npublic class GAM implements ProbabilisticClassifier, Regressor {\n\n\tclass RegressorList implements Iterable<Regressor> {\n\n\t\tList<Regressor> regressors;\n\n\t\tRegressorList() {\n\t\t\tregressors = new ArrayList<>();\n\t\t}\n\n\t\t@Override\n\t\tpublic Iterator<Regressor> iterator() {\n\t\t\treturn regressors.iterator();\n\t\t}\n\t}\n\n\tclass TermList implements Iterable<int[]> {\n\n\t\tList<int[]> terms;\n\n\t\tTermList() {\n\t\t\tterms = new ArrayList<>();\n\t\t}\n\n\t\t@Override\n\t\tpublic Iterator<int[]> iterator() {\n\t\t\treturn terms.iterator();\n\t\t}\n\t}\n\n\tprotected double intercept;\n\tprotected List<Regressor> regressors;\n\tprotected List<int[]> terms;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic GAM() {\n\t\tregressors = new ArrayList<>();\n\t\tterms = new ArrayList<>();\n\t\tintercept = 0;\n\t}\n\n\t/**\n\t * Returns the intercept.\n\t * \n\t * @return the intercept.\n\t */\n\tpublic double getIntercept() {\n\t\treturn intercept;\n\t}\n\n\t/**\n\t * Sets the intercept.\n\t * \n\t * @param intercept the new intercept.\n\t */\n\tpublic void setIntercept(double intercept) {\n\t\tthis.intercept = intercept;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tintercept = Double.parseDouble(in.readLine().split(\": \")[1]);\n\t\tint size = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tregressors = new ArrayList<>(size);\n\t\tterms = new ArrayList<>(size);\n\t\tin.readLine();\n\t\tfor (int i = 0; i < size; i++) {\n\t\t\tint[] term = ArrayUtils.parseIntArray(in.readLine().split(\": \")[1]);\n\t\t\tterms.add(term);\n\t\t\tin.readLine();\n\n\t\t\tString line = in.readLine();\n\t\t\tString regressorName = line.substring(1, line.length() - 1).split(\": \")[1];\n\t\t\tClass<?> clazz = Class.forName(regressorName);\n\t\t\tRegressor regressor = (Regressor) clazz.getDeclaredConstructor().newInstance();\n\t\t\tregressor.read(in);\n\t\t\tregressors.add(regressor);\n\t\t\tin.readLine();\n\t\t}\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Intercept: \" + intercept);\n\t\tout.println(\"Components: \" + regressors.size());\n\t\tout.println();\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tout.println(\"Component: \" + Arrays.toString(terms.get(i)));\n\t\t\tout.println();\n\t\t\tregressors.get(i).write(out);\n\t\t\tout.println();\n\t\t}\n\t}\n\n\t/**\n\t * Adds a new term into this GAM. The term is an array of attribute indices that are used in the regressor.\n\t * \n\t * @param term the new term to add.\n\t * @param regressor the new regressor to add.\n\t */\n\tpublic void add(int[] term, Regressor regressor) {\n\t\tterms.add(term);\n\t\tregressors.add(regressor);\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\tdouble pred = intercept;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tpred += regressor.regress(instance);\n\t\t}\n\t\treturn pred;\n\t}\n\n\t@Override\n\tpublic int classify(Instance instance) {\n\t\tdouble pred = regress(instance);\n\t\treturn pred >= 0 ? 1 : 0;\n\t}\n\n\t@Override\n\tpublic double[] predictProbabilities(Instance instance) {\n\t\tdouble pred = regress(instance);\n\t\tdouble prob = MathUtils.sigmoid(pred);\n\t\treturn new double[] { 1 - prob, prob };\n\t}\n\n\t/**\n\t * Returns the term list.\n\t * \n\t * @return the term list.\n\t */\n\tpublic List<int[]> getTerms() {\n\t\treturn terms;\n\t}\n\n\t/**\n\t * Returns the regressor list.\n\t * \n\t * @return the regressor list.\n\t */\n\tpublic List<Regressor> getRegressors() {\n\t\treturn regressors;\n\t}\n\n\t@Override\n\tpublic GAM copy() {\n\t\tGAM copy = new GAM();\n\t\tcopy.intercept = intercept;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tcopy.regressors.add((Regressor) regressor.copy());\n\t\t}\n\t\tfor (int[] term : terms) {\n\t\t\tcopy.terms.add(term.clone());\n\t\t}\n\n\t\treturn copy;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/GAMLearner.java",
    "content": "package mltk.predictor.gam;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.HoldoutValidatedLearnerWithTaskOptions;\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.Attribute.Type;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.BaggedEnsemble;\nimport mltk.predictor.BoostedEnsemble;\nimport mltk.predictor.HoldoutValidatedLearner;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.function.BaggedLineCutter;\nimport mltk.predictor.function.CompressionUtils;\nimport mltk.predictor.function.EnsembledLineCutter;\nimport mltk.predictor.function.Function1D;\nimport mltk.predictor.function.SubaggedLineCutter;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.OptimUtils;\nimport mltk.util.Random;\n\n/**\n * Class for learning GAMs via gradient tree boosting.\n * \n * <p>\n * Reference:<br>\n * Y. Lou, R. Caruana and J. Gehrke. Intelligible models for classification and regression. In <i>Proceedings of the\n * 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD)</i>, Beijing, China, 2012.<br>\n * \n * Y. Lou, Y. Wang, S. Liang and Y. Dong. Efficiently Training Intelligible Models for Global Explanations. \n * In <i>Proceedings of the 29th ACM International Conference on Information and Knowledge Management (CIKM)</i>, \n * Virtual Event, Ireland, 2020.\n * </p>\n * \n * @author Yin Lou\n * \n */\npublic class GAMLearner extends HoldoutValidatedLearner {\n\t\n\tstatic class Options extends HoldoutValidatedLearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-b\", description = \"base learner (default: tr:3:100:0.65)\")\n\t\tString baseLearner = \"tr:3:100:0.65\";\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t\t@Argument(name = \"-l\", description = \"learning rate (default: 0.01)\")\n\t\tdouble learningRate = 0.01;\n\n\t}\n\n\t/**\n\t * Trains a GAM.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.GAMLearner\n\t * -t\ttrain set path\n\t * -m\tmaximum number of iterations\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-v]\tvalid set path\n\t * [-e]\tevaluation metric (default: default metric of task)\n\t * [-S]\tconvergence criteria (default: -1)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-b]\tbase learner (default: tr:3:100)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * [-l]\tlearning rate (default: 0.01)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(GAMLearner.class, opts);\n\t\tTask task = null;\n\t\tMetric metric = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t\tif (opts.metric == null) {\n\t\t\t\tmetric = task.getDefaultMetric();\n\t\t\t} else {\n\t\t\t\tmetric = MetricFactory.getMetric(opts.metric);\n\t\t\t}\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\t\t\n\t\tConvergenceTester ct = ConvergenceTester.parse(opts.cc);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\t\t\n\t\tGAMLearner learner = new GAMLearner();\n\t\tlearner.setBaseLearner(opts.baseLearner);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setLearningRate(opts.learningRate);\n\t\tlearner.setTask(task);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setConvergenceTester(ct);\n\t\tlearner.setVerbose(opts.verbose);\n\n\t\tif (opts.validPath != null) {\n\t\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\t\t\tlearner.setValidSet(validSet);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGAM gam = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(gam, opts.outputModelPath);\n\t\t}\n\t}\n\n\tprivate int baggingIters;\n\tprivate int maxNumIters;\n\tprivate int maxNumLeaves;\n\tprivate Task task;\n\tprivate double alpha;\n\tprivate double learningRate;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic GAMLearner() {\n\t\tverbose = false;\n\t\tbaggingIters = 100;\n\t\tmaxNumIters = -1;\n\t\tmaxNumLeaves = 3;\n\t\talpha = 0.65;\n\t\tlearningRate = 0.01;\n\t\ttask = Task.REGRESSION;\n\t\tmetric = task.getDefaultMetric();\n\t}\n\n\t/**\n\t * Returns the number of bagging iterations.\n\t * \n\t * @return the number of bagging iterations.\n\t */\n\tpublic int getBaggingIters() {\n\t\treturn baggingIters;\n\t}\n\n\t/**\n\t * Sets the number of bagging iterations.\n\t * \n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void setBaggingIters(int baggingIters) {\n\t\tthis.baggingIters = baggingIters;\n\t}\n\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\n\t/**\n\t * Returns the maximum number of leaves.\n\t * \n\t * @return the maximum number of leaves.\n\t */\n\tpublic int getMaxNumLeaves() {\n\t\treturn maxNumLeaves;\n\t}\n\n\t/**\n\t * Sets the maximum number of leaves.\n\t * \n\t * @param maxNumLeaves the maximum number of leaves.\n\t */\n\tpublic void setMaxNumLeaves(int maxNumLeaves) {\n\t\tthis.maxNumLeaves = maxNumLeaves;\n\t}\n\n\t/**\n\t * Returns the learning rate.\n\t * \n\t * @return the learning rate.\n\t */\n\tpublic double getLearningRate() {\n\t\treturn learningRate;\n\t}\n\n\t/**\n\t * Sets the learning rate.\n\t * \n\t * @param learningRate the learning rate.\n\t */\n\tpublic void setLearningRate(double learningRate) {\n\t\tthis.learningRate = learningRate;\n\t}\n\t\n\t/**\n\t * Returns the subsampling ratio.\n\t * \n\t * @return the subsampling ratio.\n\t */\n\tpublic double getSubsamplingRatio() {\n\t\treturn alpha;\n\t}\n\t\n\t/**\n\t * Sets the subsampling ratio.\n\t * \n\t * @param alpha the subsampling ratio.\n\t */\n\tpublic void setSubsamplingRatio(double alpha) {\n\t\tthis.alpha = alpha;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\n\t/**\n\t * Sets the base learner.<br>\n\t * \n\t * @param option the option string.\n\t */\n\tpublic void setBaseLearner(String option) {\n\t\tString[] opts = option.split(\":\");\n\t\tswitch (opts[0]) {\n\t\t\tcase \"tr\":\n\t\t\t\tint maxNumLeaves = Integer.parseInt(opts[1]);\n\t\t\t\tint baggingIters = Integer.parseInt(opts[2]);\n\t\t\t\tsetMaxNumLeaves(maxNumLeaves);\n\t\t\t\tsetBaggingIters(baggingIters);\n\t\t\t\tif (opts.length > 3) {\n\t\t\t\t\tdouble alpha = Double.parseDouble(opts[3]);\n\t\t\t\t\tsetSubsamplingRatio(alpha);\n\t\t\t\t} else {\n\t\t\t\t\tsetSubsamplingRatio(-1);\n\t\t\t\t}\n\t\t\t\tbreak;\n\t\t\tcase \"cs\":\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param maxNumLeaves the maximum number of leaves.\n\t * @return a classifier.\n\t */\n\tpublic GAM buildClassifier(Instances trainSet, Instances validSet, int maxNumIters, int maxNumLeaves) {\n\t\tGAM gam = new GAM();\n\n\t\t// Backup targets and weights\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tdouble[] weight = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\ttarget[i] = instance.getTarget();\n\t\t\tweight[i] = instance.getWeight();\n\t\t}\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>(attributes.size());\n\t\tfor (int i = 0; i < attributes.size(); i++) {\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Create bags\n\t\tEnsembledLineCutter elc = null;\n\t\tif (0 <= alpha & alpha <= 1) {\n\t\t\tSubaggedLineCutter slc = new SubaggedLineCutter(true);\n\t\t\tslc.createSubags(trainSet.size(), alpha, baggingIters);\n\t\t\telc = slc;\n\t\t} else {\n\t\t\tBaggedLineCutter blc = new BaggedLineCutter(true);\n\t\t\tblc.createBags(trainSet.size(), baggingIters);\n\t\t\telc = blc;\n\t\t}\n\t\telc.setNumIntervals(maxNumLeaves);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] predTrain = new double[trainSet.size()];\n\t\tdouble[] probTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tOptimUtils.computeProbabilities(predTrain, probTrain);\n\t\tOptimUtils.computePseudoResidual(predTrain, target, rTrain);\n\t\tdouble[] pValid = new double[validSet.size()];\n\n\t\t// Gradient boosting\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t// Derivitive to attribute k\n\t\t\t\t// Minimizes the loss function: log(1 + exp(-yF))\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble prob = probTrain[i];\n\t\t\t\t\tdouble w = prob * (1 - prob);\n\t\t\t\t\tinstance.setTarget(rTrain[i] * weight[i]);\n\t\t\t\t\tinstance.setWeight(w * weight[i]);\n\t\t\t\t}\n\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\n\t\t\t\t// Train model\n\t\t\t\telc.setAttributeIndex(j);\n\t\t\t\tBaggedEnsemble baggedEnsemble = elc.build(trainSet);\n\t\t\t\tFunction1D func = CompressionUtils.compress(attributes.get(j).getIndex(), baggedEnsemble);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(func);\n\t\t\t\tbaggedEnsemble = null;\n\n\t\t\t\t// Update predictions\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\tpredTrain[i] += pred;\n\t\t\t\t}\n\t\t\t\tOptimUtils.computeProbabilities(predTrain, probTrain);\n\t\t\t\tOptimUtils.computePseudoResidual(predTrain, target, rTrain);\n\t\t\t\t\n\t\t\t\tfor (int i = 0; i < validSet.size(); i++) {\n\t\t\t\t\tInstance instance = validSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\tpValid[i] += pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\t\tct.add(measure);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" Feature \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\n\t\t// Remove trees\n\t\tint n = idx / attributes.size();\n\t\tint m = idx % attributes.size();\n\t\tfor (int k = 0; k < regressors.size(); k++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(k);\n\t\t\tfor (int i = boostedEnsemble.size(); i > n + 1; i--) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t\tif (k > m) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets and weights\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t\ttrainSet.get(i).setWeight(weight[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tAttribute attribute = attributes.get(i);\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tFunction1D function = CompressionUtils.compress(attIndex, boostedEnsemble);\n\t\t\tRegressor regressor = function;\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint l = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint l = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t}\n\t\t\tgam.add(new int[] { attIndex }, regressor);\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param maxNumLeaves the maximum number of leaves.\n\t * @return a classifier.\n\t */\n\tpublic GAM buildClassifier(Instances trainSet, int maxNumIters, int maxNumLeaves) {\n\t\tGAM gam = new GAM();\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\n\t\t// Backup targets and weights\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tdouble[] weight = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\ttarget[i] = instance.getTarget();\n\t\t\tweight[i] = instance.getWeight();\n\t\t}\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>(attributes.size());\n\t\tfor (int i = 0; i < attributes.size(); i++) {\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Create bags\n\t\tEnsembledLineCutter elc = null;\n\t\tif (0 <= alpha & alpha <= 1) {\n\t\t\tSubaggedLineCutter slc = new SubaggedLineCutter(true);\n\t\t\tslc.createSubags(trainSet.size(), alpha, baggingIters);\n\t\t\telc = slc;\n\t\t} else {\n\t\t\tBaggedLineCutter blc = new BaggedLineCutter(true);\n\t\t\tblc.createBags(trainSet.size(), baggingIters);\n\t\t\telc = blc;\n\t\t}\n\t\telc.setNumIntervals(maxNumLeaves);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] predTrain = new double[trainSet.size()];\n\t\tdouble[] probTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tOptimUtils.computeProbabilities(predTrain, probTrain);\n\t\tOptimUtils.computePseudoResidual(predTrain, target, rTrain);\n\n\t\t// Gradient boosting\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t// Derivitive to attribute k\n\t\t\t\t// Minimizes the loss function: log(1 + exp(-yF))\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble prob = probTrain[i];\n\t\t\t\t\tdouble w = prob * (1 - prob);\n\t\t\t\t\tinstance.setTarget(rTrain[i] * weight[i]);\n\t\t\t\t\tinstance.setWeight(w * weight[i]);\n\t\t\t\t}\n\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\n\t\t\t\t// Train model\n\t\t\t\telc.setAttributeIndex(j);\n\t\t\t\tBaggedEnsemble baggedEnsemble = elc.build(trainSet);\n\t\t\t\tFunction1D func = CompressionUtils.compress(attributes.get(j).getIndex(), baggedEnsemble);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(func);\n\t\t\t\tbaggedEnsemble = null;\n\n\t\t\t\t// Update predictions\n\t\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\tpredTrain[i] += pred;\n\t\t\t\t}\n\t\t\t\tOptimUtils.computeProbabilities(predTrain, probTrain);\n\t\t\t\tOptimUtils.computePseudoResidual(predTrain, target, rTrain);\n\n\t\t\t\tdouble measure = simpleMetric.eval(predTrain, target);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" Feature \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets and weights\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t\ttrainSet.get(i).setWeight(weight[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tAttribute attribute = attributes.get(i);\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tFunction1D function = CompressionUtils.compress(attIndex, boostedEnsemble);\n\t\t\tRegressor regressor = function;\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint l = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint l = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t}\n\t\t\tgam.add(new int[] { attIndex }, regressor);\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param maxNumLeaves the maximum number of leaves.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(Instances trainSet, Instances validSet, int maxNumIters, int maxNumLeaves) {\n\t\tGAM gam = new GAM();\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>(attributes.size());\n\t\tfor (int i = 0; i < attributes.size(); i++) {\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tEnsembledLineCutter elc = null;\n\t\tif (0 <= alpha & alpha <= 1) {\n\t\t\tSubaggedLineCutter slc = new SubaggedLineCutter(false);\n\t\t\tslc.createSubags(trainSet.size(), alpha, baggingIters);\n\t\t\telc = slc;\n\t\t} else {\n\t\t\tBaggedLineCutter blc = new BaggedLineCutter(false);\n\t\t\tblc.createBags(trainSet.size(), baggingIters);\n\t\t\telc = blc;\n\t\t}\n\t\telc.setNumIntervals(maxNumLeaves);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tdouble[] pValid = new double[validSet.size()];\n\t\tdouble[] rValid = new double[validSet.size()];\n\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\trTrain[i] = instance.getTarget();\n\t\t}\n\t\tfor (int i = 0; i < validSet.size(); i++) {\n\t\t\tInstance instance = validSet.get(i);\n\t\t\trValid[i] = instance.getTarget();\n\t\t}\n\n\t\t// Gradient boosting\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t// Derivative to attribute k\n\t\t\t\t// Equivalent to residual\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\t\t\t\t// Prepare training set\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\t\t\t\t// Train model\n\t\t\t\telc.setAttributeIndex(j);\n\t\t\t\tBaggedEnsemble baggedEnsemble = elc.build(trainSet);\n\t\t\t\tFunction1D func = CompressionUtils.compress(attributes.get(j).getIndex(), baggedEnsemble);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(func);\n\t\t\t\tbaggedEnsemble = null;\n\n\t\t\t\t// Update residuals\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\trTrain[i] -= pred;\n\t\t\t\t}\n\t\t\t\tfor (int i = 0; i < rValid.length; i++) {\n\t\t\t\t\tInstance instance = validSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\tpValid[i] += pred;\n\t\t\t\t\trValid[i] -= pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\t\tct.add(measure);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" Feature \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\n\t\t// Prune tree ensembles\n\t\tint n = idx / attributes.size();\n\t\tint m = idx % attributes.size();\n\t\tfor (int k = 0; k < regressors.size(); k++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(k);\n\t\t\tfor (int i = boostedEnsemble.size(); i > n + 1; i--) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t\tif (k > m) {\n\t\t\t\tboostedEnsemble.removeLast();\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tAttribute attribute = attributes.get(i);\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tFunction1D function = CompressionUtils.compress(attIndex, boostedEnsemble);\n\t\t\tRegressor regressor = function;\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint l = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint l = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t}\n\t\t\tgam.add(new int[] { attIndex }, regressor);\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param maxNumLeaves the maximum number of leaves.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(Instances trainSet, int maxNumIters, int maxNumLeaves) {\n\t\tGAM gam = new GAM();\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tList<BoostedEnsemble> regressors = new ArrayList<>(attributes.size());\n\t\tfor (int i = 0; i < attributes.size(); i++) {\n\t\t\tregressors.add(new BoostedEnsemble());\n\t\t}\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\t// Create bags\n\t\tEnsembledLineCutter elc = null;\n\t\tif (0 <= alpha & alpha <= 1) {\n\t\t\tSubaggedLineCutter slc = new SubaggedLineCutter(true);\n\t\t\tslc.createSubags(trainSet.size(), alpha, baggingIters);\n\t\t\telc = slc;\n\t\t} else {\n\t\t\tBaggedLineCutter blc = new BaggedLineCutter(true);\n\t\t\tblc.createBags(trainSet.size(), baggingIters);\n\t\t\telc = blc;\n\t\t}\n\t\telc.setNumIntervals(maxNumLeaves);\n\n\t\t// Initialize predictions and residuals\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\trTrain[i] = instance.getTarget();\n\t\t}\n\n\t\t// Gradient boosting\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t// Derivative to attribute k\n\t\t\t\t// Equivalent to residual\n\t\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(j);\n\t\t\t\t// Prepare training set\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t\t}\n\t\t\t\t// Train model\n\t\t\t\telc.setAttributeIndex(j);\n\t\t\t\tBaggedEnsemble baggedEnsemble = elc.build(trainSet);\n\t\t\t\tFunction1D func = CompressionUtils.compress(attributes.get(j).getIndex(), baggedEnsemble);\n\t\t\t\tif (learningRate != 1) {\n\t\t\t\t\tfunc.multiply(learningRate);\n\t\t\t\t}\n\t\t\t\tboostedEnsemble.add(func);\n\t\t\t\tbaggedEnsemble = null;\n\n\t\t\t\t// Update residuals\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\tdouble pred = func.regress(instance);\n\t\t\t\t\tpTrain[i] += pred;\n\t\t\t\t\trTrain[i] -= pred;\n\t\t\t\t}\n\n\t\t\t\tdouble measure = simpleMetric.eval(pTrain, target);\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \" Feature \" + j + \": \" + measure);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\t// Compress model\n\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\tBoostedEnsemble boostedEnsemble = regressors.get(i);\n\t\t\tAttribute attribute = attributes.get(i);\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tFunction1D function = CompressionUtils.compress(attIndex, boostedEnsemble);\n\t\t\tRegressor regressor = function;\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint l = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint l = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tregressor = CompressionUtils.convert(l, function);\n\t\t\t}\n\t\t\tgam.add(new int[] { attIndex }, regressor);\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\t@Override\n\tpublic GAM build(Instances instances) {\n\t\tGAM gam = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tif (metric == null) {\n\t\t\tmetric = task.getDefaultMetric();\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tif (validSet != null) {\n\t\t\t\t\tgam = buildRegressor(instances, validSet, maxNumIters, maxNumLeaves);\n\t\t\t\t} else {\n\t\t\t\t\tgam = buildRegressor(instances, maxNumIters, maxNumLeaves);\n\t\t\t\t}\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tif (validSet != null) {\n\t\t\t\t\tgam = buildClassifier(instances, validSet, maxNumIters, maxNumLeaves);\n\t\t\t\t} else {\n\t\t\t\t\tgam = buildClassifier(instances, maxNumIters, maxNumLeaves);\n\t\t\t\t}\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn gam;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/GAMUtils.java",
    "content": "package mltk.predictor.gam;\n\nimport java.util.List;\n\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.NominalAttribute;\nimport mltk.core.NumericalAttribute;\nimport mltk.predictor.function.Array1D;\nimport mltk.predictor.function.LinearFunction;\nimport mltk.predictor.glm.GLM;\n\nclass GAMUtils {\n\n\tstatic GAM getGAM(GLM glm, List<Attribute> attList) {\n\t\tdouble[] w = glm.coefficients(0);\n\t\t\n\t\tGAM gam = new GAM();\n\t\tint k = 0;\n\t\tfor (Attribute attribute : attList) {\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tint[] term = new int[] {attIndex};\n\t\t\tif (attribute instanceof NumericalAttribute) {\n\t\t\t\tLinearFunction func = new LinearFunction(attIndex, -w[k++]);\n\t\t\t\tgam.add(term, func);\n\t\t\t} else if (attribute instanceof BinnedAttribute) {\n\t\t\t\tBinnedAttribute binnedAttribute = (BinnedAttribute) attribute;\n\t\t\t\tint size = binnedAttribute.getNumBins();\n\t\t\t\tdouble[] predictions = new double[size];\n\t\t\t\tfor (int j = 0; j < predictions.length && k < w.length; j++) {\n\t\t\t\t\tpredictions[j] = -w[k++];\n\t\t\t\t}\n\t\t\t\tArray1D ary = new Array1D(attIndex, predictions);\n\t\t\t\tgam.add(term, ary);\n\t\t\t} else if (attribute instanceof NominalAttribute) {\n\t\t\t\tNominalAttribute nominalAttribute = (NominalAttribute) attribute;\n\t\t\t\tint size = nominalAttribute.getCardinality();\n\t\t\t\tdouble[] predictions = new double[size];\n\t\t\t\tfor (int j = 0; j < predictions.length && k < w.length; j++) {\n\t\t\t\t\tpredictions[j] = -w[k++];\n\t\t\t\t}\n\t\t\t\tArray1D ary = new Array1D(attIndex, predictions);\n\t\t\t\tgam.add(term, ary);\n\t\t\t}\n\t\t}\n\t\tgam.setIntercept(-glm.intercept(0));\n\t\t\n\t\treturn gam;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/SPLAMLearner.java",
    "content": "package mltk.predictor.gam;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerWithTaskOptions;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Learner;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.function.CubicSpline;\nimport mltk.predictor.function.LinearFunction;\nimport mltk.predictor.glm.GLM;\nimport mltk.predictor.glm.RidgeLearner;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.ArrayUtils;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for learning SPLAM models. Currently only cubic spline basis is supported.\n * \n * @author Yin Lou\n * \n */\npublic class SPLAMLearner extends Learner {\n\t\n\tstatic class Options extends LearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-d\", description = \"number of knots (default: 10)\")\n\t\tint numKnots = 10;\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations (default: 0)\")\n\t\tint maxNumIters = 0;\n\t\t\n\t\t@Argument(name = \"-l\", description = \"lambda (default: 0)\")\n\t\tdouble lambda = 0;\n\t\t\n\t\t@Argument(name = \"-a\", description = \"alpha (default: 1, i.e., SPAM model)\")\n\t\tdouble alpha = 1;\n\t\t\n\t\t@Argument(name = \"-L\", description = \"whether to compute lambda max for a given a\")\n\t\tboolean lambdaMax = false;\n\n\t}\n\t\n\t/**\n\t * Trains a SPLAM.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.SPLAMLearner\n\t * -t\ttrain set path\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-d]\tnumber of knots (default: 10)\n\t * [-m]\tmaximum number of iterations (default: 0)\n\t * [-l]\tlambda (default: 0)\n\t * [-a]\talpha (default: 1, i.e., SPAM model)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(SPLAMLearner.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t\tif (opts.numKnots < 0) {\n\t\t\t\tthrow new IllegalArgumentException(\"Number of knots must be positive.\");\n\t\t\t}\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tSPLAMLearner learner = new SPLAMLearner();\n\t\tlearner.setNumKnots(opts.numKnots);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setLambda(opts.lambda);\n\t\tlearner.setAlpha(opts.alpha);\n\t\tlearner.setTask(task);\n\t\tlearner.setVerbose(opts.verbose);\n\t\t\n\t\tif (opts.lambdaMax) {\n\t\t\tSystem.out.println(learner.findMaxLambda(trainSet, task, opts.numKnots, opts.alpha));\n\t\t\tSystem.exit(0);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGAM gam = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\t\t\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(gam, opts.outputModelPath);\n\t\t}\n\t}\n\n\tstatic class ModelStructure {\n\n\t\tstatic final byte ELIMINATED = 0;\n\t\tstatic final byte LINEAR = 1;\n\t\tstatic final byte NONLINEAR = 2;\n\n\t\tbyte[] structure;\n\n\t\tModelStructure(byte[] structure) {\n\t\t\tthis.structure = structure;\n\t\t}\n\n\t\t@Override\n\t\tpublic boolean equals(Object obj) {\n\t\t\tif (this == obj)\n\t\t\t\treturn true;\n\t\t\tif (obj == null)\n\t\t\t\treturn false;\n\t\t\tif (getClass() != obj.getClass())\n\t\t\t\treturn false;\n\t\t\tModelStructure other = (ModelStructure) obj;\n\t\t\tif (!Arrays.equals(structure, other.structure))\n\t\t\t\treturn false;\n\t\t\treturn true;\n\t\t}\n\n\t\t@Override\n\t\tpublic int hashCode() {\n\t\t\tfinal int prime = 31;\n\t\t\tint result = 1;\n\t\t\tresult = prime * result + Arrays.hashCode(structure);\n\t\t\treturn result;\n\t\t}\n\n\t}\n\t\n\tprivate boolean fitIntercept;\n\tprivate boolean refit;\n\tprivate int numKnots;\n\tprivate int maxNumIters;\n\tprivate Task task;\n\n\tprivate double lambda;\n\n\tprivate double alpha;\n\n\tprivate double epsilon;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic SPLAMLearner() {\n\t\tverbose = false;\n\t\tfitIntercept = true;\n\t\trefit = false;\n\t\tnumKnots = 10;\n\t\tmaxNumIters = -1;\n\t\tlambda = 0.0;\n\t\talpha = 1;\n\t\tepsilon = MathUtils.EPSILON;\n\t\ttask = Task.REGRESSION;\n\t}\n\n\t@Override\n\tpublic GAM build(Instances instances) {\n\t\tGAM gam = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tif (numKnots < 0) {\n\t\t\tnumKnots = 10;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tgam = buildRegressor(instances, maxNumIters, numKnots, lambda, alpha);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tgam = buildClassifier(instances, maxNumIters, numKnots, lambda, alpha);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn gam;\n\t}\n\n\t/**\n\t * Returns a binary classifier.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param knots the knots.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a binary classifier.\n\t */\n\tpublic GAM buildBinaryClassifier(int[] attrs, double[][][] x, double[] y, double[][] knots, int maxNumIters, double lambda,\n\t\t\tdouble alpha) {\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[x[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tdouble[] tl2 = new double[attrs.length];\n\t\tgetRegularizationParameters(lambda, alpha, tl1, tl2, y.length);\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\t\tdouble[] gamma1 = new double[m];\n\t\tdouble[] gamma2 = new double[m - 1];\n\n\t\tboolean[] activeSet = new boolean[attrs.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePass(x, y, tl1, tl2, true, activeSet, w, stepSize, \n\t\t\t\t\tg, gradient, gamma1, gamma2, pTrain, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = OptimUtils.computeLogisticLoss(pTrain, y) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePass(x, y, tl1, tl2, false, activeSet, w, stepSize, g, gradient, gamma1, gamma2, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = OptimUtils.computeLogisticLoss(pTrain, y) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tbyte[] structure = extractStructure(w);\n\t\t\tGAM gam = refitClassifier(attrs, structure, x, y, knots, w, maxNumIters);\n\t\t\treturn gam;\n\t\t} else {\n\t\t\treturn getGAM(attrs, knots, w, intercept);\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns a binary classifier.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param knots the knots.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a binary classifier.\n\t */\n\tpublic GAM buildBinaryClassifier(int[] attrs, int[][] indices, double[][][] values, double[] y, double[][] knots,\n\t\t\tint maxNumIters, double lambda, double alpha) {\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[values[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tdouble[] tl2 = new double[attrs.length];\n\t\tgetRegularizationParameters(lambda, alpha, tl1, tl2, y.length);\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\t\tdouble[] gamma1 = new double[m];\n\t\tdouble[] gamma2 = new double[m - 1];\n\t\t\n\t\tboolean[] activeSet = new boolean[attrs.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, y, rTrain);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePass(indices, values, y, tl1, tl2, true, activeSet, w, stepSize, \n\t\t\t\t\tg, gradient, gamma1, gamma2, pTrain, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = OptimUtils.computeLogisticLoss(pTrain, y) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePass(indices, values, y, tl1, tl2, false, activeSet, w, stepSize, g, gradient, gamma1, gamma2, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = OptimUtils.computeLogisticLoss(pTrain, y) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tbyte[] structure = extractStructure(w);\n\t\t\tGAM gam = refitClassifier(attrs, structure, indices, values, y, knots, w, maxNumIters * 10);\n\t\t\treturn gam;\n\t\t} else {\n\t\t\treturn getGAM(attrs, knots, w, intercept);\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param numKnots the number of knots.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a classifier.\n\t */\n\tpublic GAM buildClassifier(Instances trainSet, boolean isSparse, int numKnots, int maxNumIters, double lambda,\n\t\t\tdouble alpha) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, false);\n\t\t\tSparseDesignMatrix sm = SparseDesignMatrix.createCubicSplineDesignMatrix(trainSet.size(), sd.indices,\n\t\t\t\t\tsd.values, sd.stdList, numKnots);\n\t\t\tdouble[] y = sd.y;\n\t\t\tint[][] indices = sm.indices;\n\t\t\tdouble[][][] values = sm.values;\n\t\t\tdouble[][] knots = sm.knots;\n\t\t\tint[] attrs = sd.attrs;\n\n\t\t\t// Mapping from attribute index to index in design matrix\n\t\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\t\tfor (int j = 0; j < sd.attrs.length; j++) {\n\t\t\t\tmap.put(attrs[j], j);\n\t\t\t}\n\t\t\t\n\t\t\tGAM gam = buildBinaryClassifier(attrs, indices, values, y, knots, maxNumIters, lambda, alpha);\n\n\t\t\t// Rescale weights in gam\n\t\t\tList<Regressor> regressors = gam.getRegressors();\n\t\t\tList<int[]> terms = gam.getTerms();\n\t\t\tdouble intercept = gam.getIntercept();\n\t\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\t\tRegressor regressor = regressors.get(i);\n\t\t\t\tint attIndex = terms.get(i)[0];\n\t\t\t\tint idx = map.get(attIndex);\n\t\t\t\tdouble[] std = sm.std[idx];\n\t\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\t\tfunc.setSlope(func.getSlope() / std[0]);\n\t\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\t\tdouble[] w = spline.getCoefficients();\n\t\t\t\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\t\t\t\tw[j] /= std[j];\n\t\t\t\t\t}\n\t\t\t\t\tdouble[] k = spline.getKnots();\n\t\t\t\t\tfor (int j = 0; j < k.length; j++) {\n\t\t\t\t\t\tintercept -= w[j + 3] * CubicSpline.h(0, k[j]);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tgam.setIntercept(intercept);\n\t\t\t}\n\n\t\t\treturn gam;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, false);\n\t\t\tDenseDesignMatrix dm = DenseDesignMatrix.createCubicSplineDesignMatrix(dd.x, dd.stdList, numKnots);\n\t\t\tdouble[] y = dd.y;\n\t\t\tdouble[][][] x = dm.x;\n\t\t\tdouble[][] knots = dm.knots;\n\t\t\tint[] attrs = dd.attrs;\n\t\t\t\n\t\t\t// Mapping from attribute index to index in design matrix\n\t\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\t\tfor (int j = 0; j < dd.attrs.length; j++) {\n\t\t\t\tmap.put(dd.attrs[j], j);\n\t\t\t}\n\n\t\t\tGAM gam = buildBinaryClassifier(attrs, x, y, knots, maxNumIters, lambda, alpha);\n\n\t\t\t// Rescale weights in gam\n\t\t\tList<Regressor> regressors = gam.getRegressors();\n\t\t\tList<int[]> terms = gam.getTerms();\n\t\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\t\tRegressor regressor = regressors.get(i);\n\t\t\t\tint attIndex = terms.get(i)[0];\n\t\t\t\tint idx = map.get(attIndex);\n\t\t\t\tdouble[] std = dm.std[idx];\n\t\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\t\tfunc.setSlope(func.getSlope() / std[0]);\n\t\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\t\tdouble[] w = spline.getCoefficients();\n\t\t\t\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\t\t\t\tw[j] /= std[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn gam;\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numKnots the number of knots.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a classifier.\n\t */\n\tpublic GAM buildClassifier(Instances trainSet, int maxNumIters, int numKnots, double lambda, double alpha) {\n\t\treturn buildClassifier(trainSet, isSparse(trainSet), maxNumIters, numKnots, lambda, alpha);\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numKnots the number of knots.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(Instances trainSet, boolean isSparse, int maxNumIters, int numKnots, double lambda,\n\t\t\tdouble alpha) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, false);\n\t\t\tSparseDesignMatrix sm = SparseDesignMatrix.createCubicSplineDesignMatrix(trainSet.size(), sd.indices,\n\t\t\t\t\tsd.values, sd.stdList, numKnots);\n\t\t\tdouble[] y = sd.y;\n\t\t\tint[][] indices = sm.indices;\n\t\t\tdouble[][][] values = sm.values;\n\t\t\tdouble[][] knots = sm.knots;\n\t\t\tint[] attrs = sd.attrs;\n\n\t\t\t// Mapping from attribute index to index in design matrix\n\t\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\t\tfor (int j = 0; j < sd.attrs.length; j++) {\n\t\t\t\tmap.put(attrs[j], j);\n\t\t\t}\n\t\t\t\n\t\t\tGAM gam = buildRegressor(attrs, indices, values, y, knots, maxNumIters, lambda, alpha);\n\n\t\t\t// Rescale weights in gam\n\t\t\tList<Regressor> regressors = gam.getRegressors();\n\t\t\tList<int[]> terms = gam.getTerms();\n\t\t\tdouble intercept = gam.getIntercept();\n\t\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\t\tRegressor regressor = regressors.get(i);\n\t\t\t\tint attIndex = terms.get(i)[0];\n\t\t\t\tint idx = map.get(attIndex);\n\t\t\t\tdouble[] std = sm.std[idx];\n\t\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\t\tfunc.setSlope(func.getSlope() / std[0]);\n\t\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\t\tdouble[] w = spline.getCoefficients();\n\t\t\t\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\t\t\t\tw[j] /= std[j];\n\t\t\t\t\t}\n\t\t\t\t\tdouble[] k = spline.getKnots();\n\t\t\t\t\tfor (int j = 0; j < k.length; j++) {\n\t\t\t\t\t\tintercept -= w[j + 3] * CubicSpline.h(0, k[j]);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tgam.setIntercept(intercept);\n\t\t\t}\n\n\t\t\treturn gam;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, false);\n\t\t\tDenseDesignMatrix dm = DenseDesignMatrix.createCubicSplineDesignMatrix(dd.x, dd.stdList, numKnots);\n\t\t\tdouble[] y = dd.y;\n\t\t\tdouble[][][] x = dm.x;\n\t\t\tdouble[][] knots = dm.knots;\n\t\t\tint[] attrs = dd.attrs;\n\t\t\t\n\t\t\t// Mapping from attribute index to index in design matrix\n\t\t\tMap<Integer, Integer> map = new HashMap<>();\n\t\t\tfor (int j = 0; j < dd.attrs.length; j++) {\n\t\t\t\tmap.put(dd.attrs[j], j);\n\t\t\t}\n\n\t\t\tGAM gam = buildRegressor(attrs, x, y, knots, maxNumIters, lambda, alpha);\n\n\t\t\t// Rescale weights in gam\n\t\t\tList<Regressor> regressors = gam.getRegressors();\n\t\t\tList<int[]> terms = gam.getTerms();\n\t\t\tfor (int i = 0; i < regressors.size(); i++) {\n\t\t\t\tRegressor regressor = regressors.get(i);\n\t\t\t\tint attIndex = terms.get(i)[0];\n\t\t\t\tint idx = map.get(attIndex);\n\t\t\t\tdouble[] std = dm.std[idx];\n\t\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\t\tfunc.setSlope(func.getSlope() / std[0]);\n\t\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\t\tdouble[] w = spline.getCoefficients();\n\t\t\t\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\t\t\t\tw[j] /= std[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn gam;\n\t\t}\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numKnots the number of knots.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(Instances trainSet, int maxNumIters, int numKnots, double lambda,\n\t\t\tdouble alpha) {\n\t\treturn buildRegressor(trainSet, isSparse(trainSet), maxNumIters, numKnots, lambda, alpha);\n\t}\n\n\t/**\n\t * Returns a regressor.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param knots the knots.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(int[] attrs, double[][][] x, double[] y, double[][] knots, int maxNumIters,\n\t\t\tdouble lambda, double alpha) {\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[x[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tdouble[] tl2 = new double[attrs.length];\n\t\tgetRegularizationParameters(lambda, alpha, tl1, tl2, y.length);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t);\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\t\tdouble[] gamma1 = new double[m];\n\t\tdouble[] gamma2 = new double[m - 1];\n\n\t\tboolean[] activeSet = new boolean[attrs.length];\n\n\t\tdouble intercept = 0;\n\t\t\t\t\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePass(x, tl1, tl2, true, activeSet, w, stepSize, g, gradient, gamma1, gamma2, \n\t\t\t\t\trTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = OptimUtils.computeQuadraticLoss(rTrain) + getPenalty(w, tl1, tl2);\n\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tdoOnePass(x, tl1, tl2, false, activeSet, w, stepSize, g, gradient, gamma1, gamma2, rTrain);\n\t\t\t\t\n\t\t\t\tdouble currLoss = OptimUtils.computeQuadraticLoss(rTrain) + getPenalty(w, tl1, tl2);\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tbyte[] struct = extractStructure(w);\n\t\t\treturn refitRegressor(attrs, struct, x, y, knots, w, maxNumIters);\n\t\t} else {\n\t\t\treturn getGAM(attrs, knots, w, intercept);\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns a regressor.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param knots the knots.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param alpha the alpha.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(int[] attrs, int[][] indices, double[][][] values, double[] y, double[][] knots,\n\t\t\tint maxNumIters, double lambda, double alpha) {\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\t\t\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[values[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tdouble[] tl2 = new double[attrs.length];\n\t\tgetRegularizationParameters(lambda, alpha, tl1, tl2, y.length);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = 0;\n\t\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\t\tl += y[index[j]] * t[i];\n\t\t\t\t}\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\t\tdouble[] gamma1 = new double[m];\n\t\tdouble[] gamma2 = new double[m - 1];\n\t\t\n\t\tboolean[] activeSet = new boolean[attrs.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePass(indices, values, tl1, tl2, true, activeSet, w, stepSize, \n\t\t\t\t\tg, gradient, gamma1, gamma2, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = OptimUtils.computeQuadraticLoss(rTrain) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePass(indices, values, tl1, tl2, false, activeSet, w, stepSize, g, gradient, gamma1, gamma2, rTrain);\n\n\t\t\t\tdouble currLoss = OptimUtils.computeQuadraticLoss(rTrain) + getPenalty(w, tl1, tl2);\n\t\t\t\t\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tbyte[] structure = extractStructure(w);\n\t\t\tGAM gam = refitRegressor(attrs, structure, indices, values, y, knots, w, maxNumIters * 10);\n\t\t\treturn gam;\n\t\t} else {\n\t\t\treturn getGAM(attrs, knots, w, intercept);\n\t\t}\n\t}\n\n\t/**\n\t * Returns {@code true} if we fit intercept.\n\t * \n\t * @return {@code true} if we fit intercept.\n\t */\n\tpublic boolean fitIntercept() {\n\t\treturn fitIntercept;\n\t}\n\n\t/**\n\t * Sets whether we fit intercept.\n\t * \n\t * @param fitIntercept whether we fit intercept.\n\t */\n\tpublic void fitIntercept(boolean fitIntercept) {\n\t\tthis.fitIntercept = fitIntercept;\n\t}\n\n\t/**\n\t * Returns the alpha.\n\t * \n\t * @return the alpha;\n\t */\n\tpublic double getAlpha() {\n\t\treturn alpha;\n\t}\n\t\n\t/**\n\t * Returns the convergence threshold epsilon.\n\t * \n\t * @return the convergence threshold epsilon.\n\t */\n\tpublic double getEpsilon() {\n\t\treturn epsilon;\n\t}\n\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\t\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\n\t/**\n\t * Returns the number of knots.\n\t * \n\t * @return the number of knots.\n\t */\n\tpublic int getNumKnots() {\n\t\treturn numKnots;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Returns {@code true} if we output something during the training.\n\t * \n\t * @return {@code true} if we output something during the training.\n\t */\n\tpublic boolean isVerbose() {\n\t\treturn verbose;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if we refit the model.\n\t * \n\t * @return {@code true} if we refit the model.\n\t */\n\tpublic boolean refit() {\n\t\treturn refit;\n\t}\n\n\t/**\n\t * Sets whether we refit the model.\n\t * \n\t * @param refit {@code true} if we refit the model.\n\t */\n\tpublic void refit(boolean refit) {\n\t\tthis.refit = refit;\n\t}\n\n\t/**\n\t * Sets the alpha.\n\t * \n\t * @param alpha the alpha.\n\t */\n\tpublic void setAlpha(double alpha) {\n\t\tthis.alpha = alpha;\n\t}\n\n\t/**\n\t * Sets the convergence threshold epsilon.\n\t * \n\t * @param epsilon the convergence threshold epsilon.\n\t */\n\tpublic void setEpsilon(double epsilon) {\n\t\tthis.epsilon = epsilon;\n\t}\n\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\t\n\t/**\n\t * Sets the number of knots.\n\t * \n\t * @param numKnots the new number of knots.\n\t */\n\tpublic void setNumKnots(int numKnots) {\n\t\tthis.numKnots = numKnots;\n\t}\n\t\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\t\n\t/**\n\t * Sets whether we output something during the training.\n\t * \n\t * @param verbose the switch if we output things during training.\n\t */\n\tpublic void setVerbose(boolean verbose) {\n\t\tthis.verbose = verbose;\n\t}\n\t\n\tpublic double findMaxLambda(Instances trainSet, Task task, int numKnots, double alpha) {\n\t\tDenseDataset dd = getDenseDataset(trainSet, false);\n\t\tDenseDesignMatrix dm = DenseDesignMatrix.createCubicSplineDesignMatrix(dd.x, dd.stdList, numKnots);\n\t\tdouble[] y = dd.y;\n\t\tdouble[][][] x = dm.x;\n\t\tint[] attrs = dd.attrs;\n\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[x[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tdouble[] tl2 = new double[attrs.length];\n\t\t\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\t\tdouble[] gamma1 = new double[m];\n\t\tdouble[] gamma2 = new double[m - 1];\n\n\t\tif (task == Task.REGRESSION) {\n\t\t\treturn findMaxLambda(x, y, alpha, tl1, tl2, w, g, gradient, gamma1, gamma2);\n\t\t} else {\n\t\t\tdouble[] pTrain = new double[y.length];\n\t\t\tdouble[] rTrain = new double[y.length];\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t\treturn findMaxLambda(x, y, pTrain, rTrain, alpha, tl1, tl2, w, g, gradient, gamma1, gamma2);\n\t\t}\n\t}\n\t\n\tprotected double findMaxLambda(double[][][] x, double[] rTrain, double alpha, double[] tl1, double[] tl2,\n\t\t\tdouble[][] w, double[] g, double[] gradient, double[] gamma1, double[] gamma2) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(rTrain);\n\t\t}\n\n\t\tdouble lHigh = 0;\n\t\tfor (double[][] block : x) {\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > lHigh) {\n\t\t\t\tlHigh = t;\n\t\t\t}\n\t\t}\n\t\tlHigh /= alpha;\n\t\tdouble lLow = 0;\n\t\twhile (lHigh - lLow > MathUtils.EPSILON) {\n\t\t\tdouble lambda = (lHigh + lLow) / 2;\n\t\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\t\ttl1[j] = lambda * alpha * Math.sqrt(w[j].length);\n\t\t\t\ttl2[j] = lambda * (1 - alpha) * Math.sqrt(w[j].length - 1);\n\t\t\t}\n\t\t\tboolean isZeroPoint = testZeroPoint(x, rTrain, tl1, tl2, w, g, gradient, gamma1, gamma2);\n\t\t\tif (isZeroPoint) {\n\t\t\t\tlHigh = lambda;\n\t\t\t} else {\n\t\t\t\tlLow = lambda;\n\t\t\t}\n\t\t}\n\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(rTrain, mean);\n\t\t}\n\n\t\treturn lHigh;\n\t}\n\n\tprotected double findMaxLambda(double[][][] x, double[] y, double[] pTrain, double[] rTrain, double alpha, double[] tl1,\n\t\t\tdouble[] tl2, double[][] coefficients, double[] g, double[] gradient, double[] gamma1, double[] gamma2) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\n\t\tdouble lHigh = 0;\n\t\tfor (double[][] block : x) {\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > lHigh) {\n\t\t\t\tlHigh = t;\n\t\t\t}\n\t\t}\n\t\tlHigh /= alpha;\n\t\tdouble lLow = 0;\n\t\twhile (lHigh - lLow > MathUtils.EPSILON) {\n\t\t\tdouble lambda = (lHigh + lLow) / 2;\n\t\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\t\ttl1[j] = lambda * alpha * Math.sqrt(coefficients[j].length);\n\t\t\t\ttl2[j] = lambda * (1 - alpha) * Math.sqrt(coefficients[j].length - 1);\n\t\t\t}\n\t\t\tboolean isZeroPoint = testZeroPoint(x, y, pTrain, rTrain, tl1, tl2, coefficients, g, gradient, gamma1, gamma2);\n\t\t\tif (isZeroPoint) {\n\t\t\t\tlHigh = lambda;\n\t\t\t} else {\n\t\t\t\tlLow = lambda;\n\t\t\t}\n\t\t}\n\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t}\n\n\t\treturn lHigh;\n\t}\n\t\n\tprotected boolean testZeroPoint(double[][][] x, double[] y, double[] tl1, double[] tl2, double[][] w,\n\t\t\tdouble[] g, double[] gradient, double[] gamma1, double[] gamma2) {\n\t\tfor (int k = 0; k < x.length; k++) {\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tfinal double lambda1 = tl1[k];\n\t\t\tfinal double lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, y, gradient);\n\n\t\t\tdouble[] beta = w[k];\n\t\t\tfor (int i = 0; i < beta.length; i++) {\n\t\t\t\tg[i] = gradient[i];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\t\tgamma2[i - 1] = g[i];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = VectorUtils.l2norm(gamma2);\n\t\t\t\tdouble t2 = lambda2;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tgamma1[i] = g[i] - gamma2[i - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\t\t\tif (!ArrayUtils.isConstant(g, 0, beta.length, 0)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\n\t\treturn true;\n\t}\n\n\tprotected boolean testZeroPoint(double[][][] x, double[] y, double[] pTrain, double[] rTrain, double[] tl1, \n\t\t\tdouble[] tl2, double[][] coefficients, double[] g, double[] gradient, double[] gamma1, double[] gamma2) {\n\t\tfor (int k = 0; k < x.length; k++) {\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tfinal double lambda1 = tl1[k];\n\t\t\tfinal double lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\n\t\t\tdouble[] beta = coefficients[k];\n\t\t\tfor (int i = 0; i < beta.length; i++) {\n\t\t\t\tg[i] = gradient[i];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\t\tgamma2[i - 1] = g[i];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = VectorUtils.l2norm(gamma2);\n\t\t\t\tdouble t2 = lambda2;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tgamma1[i] = g[i] - gamma2[i - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\t\t\tif (!ArrayUtils.isConstant(g, 0, beta.length, 0)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\n\t\treturn true;\n\t}\n\n\tprotected void computeGradient(double[][] block, double[] rTrain, double[] gradient) {\n\t\tfor (int i = 0; i < block.length; i++) {\n\t\t\tgradient[i] = VectorUtils.dotProduct(block[i], rTrain);\n\t\t}\n\t}\n\n\tprotected void computeGradient(int[] index, double[][] block, double[] rTrain, double[] gradient) {\n\t\tfor (int j = 0; j < block.length; j++) {\n\t\t\tdouble[] t = block[j];\n\t\t\tgradient[j] = 0;\n\t\t\tfor (int i = 0; i < t.length; i++) {\n\t\t\t\tgradient[j] += rTrain[index[i]] * t[i];\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected boolean doOnePass(double[][][] x, double[] tl1, double[] tl2, boolean isFullPass, boolean[] activeSet,\n\t\t\tdouble[][] w, double[] stepSize, double[] g, double[] gradient, double[] gamma1, double[] gamma2, \n\t\t\tdouble[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < x.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\t\t\tdouble lambda1 = tl1[k];\n\t\t\tdouble lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\t\tgamma2[i - 1] = g[i];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = Math.sqrt(StatUtils.sumSq(gamma2, 0, beta.length - 1));\n\t\t\t\tdouble t2 = lambda2 * tk;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tgamma1[i] = g[i] - gamma2[i - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1 * tk;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\n\t\t\t// Update residuals\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = beta[j] - g[j];\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\trTrain[i] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected boolean doOnePass(double[][][] x, double[] y, double[] tl1, double[] tl2, boolean isFullPass, \n\t\t\tboolean[] activeSet, double[][] w, double[] stepSize, double[] g, double[] gradient, double[] gamma1, \n\t\t\tdouble[] gamma2, double[] pTrain, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < x.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\t\t\tdouble lambda1 = tl1[k];\n\t\t\tdouble lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\t\tgamma2[i - 1] = g[i];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = Math.sqrt(StatUtils.sumSq(gamma2, 0, beta.length - 1));\n\t\t\t\tdouble t2 = lambda2 * tk;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tgamma1[i] = g[i] - gamma2[i - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1 * tk;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = g[j] - beta[j];\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tpTrain[i] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected boolean doOnePass(int[][] indices, double[][][] values, double[] tl1, double[] tl2, boolean isFullPass,\n\t\t\tboolean[] activeSet, double[][] w, double[] stepSize, double[] g, double[] gradient, double[] gamma1, \n\t\t\tdouble[] gamma2, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = values[k];\n\t\t\tint[] index = indices[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\t\t\tdouble lambda1 = tl1[k];\n\t\t\tdouble lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\t\tgamma2[i - 1] = g[i];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = Math.sqrt(StatUtils.sumSq(gamma2, 0, beta.length - 1));\n\t\t\t\tdouble t2 = lambda2 * tk;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tgamma1[i] = g[i] - gamma2[i - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1 * tk;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\t\t\t\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = beta[j] - g[j];\n\t\t\t\tfor (int i = 0; i < t.length; i++) {\n\t\t\t\t\trTrain[index[i]] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected boolean doOnePass(int[][] indices, double[][][] values, double[] y, double[] tl1, double[] tl2,\n\t\t\tboolean isFullPass, boolean[] activeSet, double[][] w, double[] stepSize, double[] g, double[] gradient,\n\t\t\tdouble[] gamma1, double[] gamma2, double[] pTrain, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tint[] index = indices[k];\n\t\t\tdouble[][] block = values[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\t\t\tdouble lambda1 = tl1[k];\n\t\t\tdouble lambda2 = tl2[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\t// Dual method\n\t\t\tif (beta.length > 1) {\n\t\t\t\tfor (int j = 1; j < beta.length; j++) {\n\t\t\t\t\tgamma2[j - 1] = g[j];\n\t\t\t\t}\n\t\t\t\tdouble norm2 = Math.sqrt(StatUtils.sumSq(gamma2, 0, beta.length - 1));\n\t\t\t\tdouble t2 = lambda2 * tk;\n\t\t\t\tif (norm2 > t2) {\n\t\t\t\t\tVectorUtils.multiply(gamma2, t2 / norm2);\n\t\t\t\t}\n\t\t\t}\n\t\t\tgamma1[0] = g[0];\n\t\t\tfor (int j = 1; j < beta.length; j++) {\n\t\t\t\tgamma1[j] = g[j] - gamma2[j - 1];\n\t\t\t}\n\t\t\tdouble norm1 = Math.sqrt(StatUtils.sumSq(gamma1, 0, beta.length));\n\t\t\tdouble t1 = lambda1 * tk;\n\t\t\tif (norm1 > t1) {\n\t\t\t\tVectorUtils.multiply(gamma1, t1 / norm1);\n\t\t\t}\n\t\t\tg[0] -= gamma1[0];\n\t\t\tfor (int i = 1; i < beta.length; i++) {\n\t\t\t\tg[i] -= (gamma1[i] + gamma2[i - 1]);\n\t\t\t}\n\t\t\t\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] value = block[j];\n\t\t\t\tdouble delta = g[j] - beta[j];\n\t\t\t\tfor (int i = 0; i < value.length; i++) {\n\t\t\t\t\tpTrain[index[i]] += delta * value[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\tfor (int idx : index) {\n\t\t\t\trTrain[idx] = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected byte[] extractStructure(double[][] w) {\n\t\tbyte[] structure = new byte[w.length];\n\t\tfor (int i = 0; i < structure.length; i++) {\n\t\t\tdouble[] beta = w[i];\n\t\t\tboolean isLinear = beta.length == 1 || ArrayUtils.isConstant(beta, 1, beta.length, 0);\n\t\t\tif (isLinear) {\n\t\t\t\tif (beta[0] != 0) {\n\t\t\t\t\tstructure[i] = ModelStructure.LINEAR;\n\t\t\t\t} else {\n\t\t\t\t\tstructure[i] = ModelStructure.ELIMINATED;\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tstructure[i] = ModelStructure.NONLINEAR;\n\t\t\t}\n\t\t}\n\t\treturn structure;\n\t}\n\n\tprotected GAM getGAM(int[] attrs, double[][] knots, double[][] w, double intercept) {\n\t\tGAM gam = new GAM();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tint attIndex = attrs[j];\n\t\t\tdouble[] beta = w[j];\n\t\t\tboolean isLinear = beta.length == 1 || ArrayUtils.isConstant(beta, 1, beta.length, 0);\n\t\t\tif (isLinear) {\n\t\t\t\tif (beta[0] != 0) {\n\t\t\t\t\t// To rule out a feature, it has to be \"linear\" and 0 slope.\n\t\t\t\t\tgam.add(new int[] { attIndex }, new LinearFunction(attIndex, beta[0]));\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tdouble[] coef = Arrays.copyOf(beta, beta.length);\n\t\t\t\tCubicSpline spline = new CubicSpline(attIndex, 0, knots[j], coef);\n\t\t\t\tgam.add(new int[] { attIndex }, spline);\n\t\t\t}\n\t\t}\n\t\tgam.setIntercept(intercept);\n\t\treturn gam;\n\t}\n\t\n\tprotected double getPenalty(double[] w, double lambda1, double lambda2) {\n\t\tdouble penalty = 0;\n\t\tdouble sumSq = StatUtils.sumSq(w);\n\t\tdouble norm1 = Math.sqrt(sumSq);\n\t\tpenalty += lambda1 * norm1;\n\t\tdouble norm2 = sumSq - w[0] * w[0];\n\t\tnorm2 = Math.sqrt(norm2);\n\t\tpenalty += lambda2 * norm2;\n\t\treturn penalty;\n\t}\n\n\tprotected double getPenalty(double[][] coef, double[] lambda1, double[] lambda2) {\n\t\tdouble penalty = 0;\n\t\tfor (int i = 0; i < coef.length; i++) {\n\t\t\tpenalty += getPenalty(coef[i], lambda1[i], lambda2[i]);\n\t\t}\n\t\treturn penalty;\n\t}\n\n\tprotected void getRegularizationParameters(double lambda, double alpha, double[] tl1, double[] tl2, int n) {\n\t\tfor (int j = 0; j < tl1.length; j++) {\n\t\t\ttl1[j] = lambda * alpha * n;\n\t\t\ttl2[j] = lambda * (1 - alpha) * n;\n\t\t}\n\t}\n\n\tprotected GAM refitClassifier(int[] attrs, byte[] struct, double[][][] x, double[] y, double[][] knots, \n\t\t\tdouble[][] w, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int i = 0; i < struct.length; i++) {\n\t\t\tif (struct[i] == ModelStructure.NONLINEAR) {\n\t\t\t\tdouble[][] t = x[i];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\txList.add(t[j]);\n\t\t\t\t}\n\t\t\t} else if (struct[i] == ModelStructure.LINEAR) {\n\t\t\t\txList.add(x[i][0]);\n\t\t\t}\n\t\t}\n\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\n\t\tint[] attrsNew = new int[xNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildBinaryClassifier(attrsNew, xNew, y, maxNumIters, 1e-8);\n\t\t\n\t\tGAM gam = getGAM(attrs, knots, w, glm.intercept(0));\n\t\tList<Regressor> regressors = gam.regressors;\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\tfunc.setSlope(coef[k++]);\n\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\tdouble[] beta = spline.getCoefficients();\n\t\t\t\tfor (int i = 0; i < beta.length; i++) {\n\t\t\t\t\tbeta[i] = coef[k++];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\tprotected GAM refitClassifier(int[] attrs, byte[] struct, int[][] indices, double[][][] values, double[] y, \n\t\t\tdouble[][] knots, double[][] w, int maxNumIters) {\n\t\tList<int[]> iList = new ArrayList<>();\n\t\tList<double[]> vList = new ArrayList<>();\n\t\tfor (int i = 0; i < struct.length; i++) {\n\t\t\tint[] index = indices[i];\n\t\t\tif (struct[i] == ModelStructure.NONLINEAR) {\n\t\t\t\tdouble[][] t = values[i];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\tiList.add(index);\n\t\t\t\t\tvList.add(t[j]);\n\t\t\t\t}\n\t\t\t} else if (struct[i] == ModelStructure.LINEAR) {\n\t\t\t\tiList.add(index);\n\t\t\t\tvList.add(values[i][0]);\n\t\t\t}\n\t\t}\n\n\t\tint[][] iNew = new int[iList.size()][];\n\t\tfor (int i = 0; i < iNew.length; i++) {\n\t\t\tiNew[i] = iList.get(i);\n\t\t}\n\t\tdouble[][] vNew = new double[vList.size()][];\n\t\tfor (int i = 0; i < vNew.length; i++) {\n\t\t\tvNew[i] = vList.get(i);\n\t\t}\n\n\t\tint[] attrsNew = new int[iNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildBinaryClassifier(attrsNew, iNew, vNew, y, maxNumIters, 1e-8);\n\t\t\n\t\tGAM gam = getGAM(attrs, knots, w, glm.intercept(0));\n\t\tList<Regressor> regressors = gam.regressors;\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\tfunc.setSlope(coef[k++]);\n\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\tdouble[] beta = spline.getCoefficients();\n\t\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\t\tbeta[j] = coef[k++];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn gam;\n\t}\n\t\n\tprotected GAM refitRegressor(int[] attrs, byte[] struct, double[][][] x, double[] y, double[][] knots,\n\t\t\tdouble[][] w, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int i = 0; i < struct.length; i++) {\n\t\t\tif (struct[i] == ModelStructure.NONLINEAR) {\n\t\t\t\tdouble[][] t = x[i];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\txList.add(t[j]);\n\t\t\t\t}\n\t\t\t} else if (struct[i] == ModelStructure.LINEAR) {\n\t\t\t\txList.add(x[i][0]);\n\t\t\t}\n\t\t}\n\n\t\tif (xList.size() == 0) {\n\t\t\tif (fitIntercept) {\n\t\t\t\tdouble intercept = StatUtils.mean(y);\n\t\t\t\tGAM gam = new GAM();\n\t\t\t\tgam.setIntercept(intercept);\n\t\t\t\treturn gam;\n\t\t\t} else {\n\t\t\t\treturn new GAM();\n\t\t\t}\n\t\t}\n\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\n\t\tint[] attrsNew = new int[xNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildGaussianRegressor(attrsNew, xNew, y, maxNumIters, 1e-8);\n\t\t\n\t\tGAM gam = getGAM(attrs, knots, w, glm.intercept(0));\n\t\tList<Regressor> regressors = gam.regressors;\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\tfunc.setSlope(coef[k++]);\n\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\tdouble[] beta = spline.getCoefficients();\n\t\t\t\tfor (int i = 0; i < beta.length; i++) {\n\t\t\t\t\tbeta[i] = coef[k++];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn gam;\n\t}\n\n\tprotected GAM refitRegressor(int[] attrs, byte[] struct, int[][] indices, double[][][] values, double[] y,\n\t\t\tdouble[][] knots, double[][] w, int maxNumIters) {\n\t\tList<int[]> iList = new ArrayList<>();\n\t\tList<double[]> vList = new ArrayList<>();\n\t\tfor (int i = 0; i < struct.length; i++) {\n\t\t\tint[] index = indices[i];\n\t\t\tif (struct[i] == ModelStructure.NONLINEAR) {\n\t\t\t\tdouble[][] t = values[i];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\tiList.add(index);\n\t\t\t\t\tvList.add(t[j]);\n\t\t\t\t}\n\t\t\t} else if (struct[i] == ModelStructure.LINEAR) {\n\t\t\t\tiList.add(index);\n\t\t\t\tvList.add(values[i][0]);\n\t\t\t}\n\t\t}\n\n\t\tint[][] iNew = new int[iList.size()][];\n\t\tfor (int i = 0; i < iNew.length; i++) {\n\t\t\tiNew[i] = iList.get(i);\n\t\t}\n\t\tdouble[][] vNew = new double[vList.size()][];\n\t\tfor (int i = 0; i < vNew.length; i++) {\n\t\t\tvNew[i] = vList.get(i);\n\t\t}\n\n\t\tint[] attrsNew = new int[iNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildGaussianRegressor(attrsNew, iNew, vNew, y, maxNumIters, 1e-8);\n\t\t\n\t\tGAM gam = getGAM(attrs, knots, w, glm.intercept(0));\n\t\tList<Regressor> regressors = gam.regressors;\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (Regressor regressor : regressors) {\n\t\t\tif (regressor instanceof LinearFunction) {\n\t\t\t\tLinearFunction func = (LinearFunction) regressor;\n\t\t\t\tfunc.setSlope(coef[k++]);\n\t\t\t} else if (regressor instanceof CubicSpline) {\n\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\tdouble[] beta = spline.getCoefficients();\n\t\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\t\tbeta[j] = coef[k++];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn gam;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/ScorecardModelLearner.java",
    "content": "package mltk.predictor.gam;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerWithTaskOptions;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.core.processor.OneHotEncoder;\nimport mltk.predictor.Learner;\nimport mltk.predictor.glm.GLM;\nimport mltk.predictor.glm.RidgeLearner;\nimport mltk.predictor.io.PredictorWriter;\n\n/**\n * Class for learning scorecard models. Scorecard models are a special kind of\n * generalized additive models where scores for each state of the feature are \n * learned. \n * \n * @author Yin Lou\n *\n */\npublic class ScorecardModelLearner extends Learner {\n\t\n\tstatic class Options extends LearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-l\", description = \"lambda (default: 0)\")\n\t\tdouble lambda = 0;\n\n\t}\n\n\t/**\n\t * Trains a scorecard model.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.ScorecardModelLearner\n\t * -t\ttrain set path\n\t * -m\tmaximum number of iterations\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-l]\tlambda (default: 0)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(ScorecardModelLearner.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tScorecardModelLearner learner = new ScorecardModelLearner();\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setLambda(opts.lambda);\n\t\tlearner.setTask(task);\n\t\tlearner.setVerbose(opts.verbose);\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGAM gam = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(gam, opts.outputModelPath);\n\t\t}\n\t}\n\t\n\tprivate int maxNumIters;\n\tprivate double lambda;\n\tprivate Task task;\n\tprivate OneHotEncoder encoder;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic ScorecardModelLearner() {\n\t\tverbose = false;\n\t\tmaxNumIters = -1;\n\t\tencoder = new OneHotEncoder();\n\t\tlambda = 0;\n\t\ttask = Task.REGRESSION;\n\t}\n\t\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\t\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\t\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\t\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\t\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\t\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the L2 regularization parameter.\n\t * @return a classifier.\n\t */\n\tpublic GAM buildClassifier(Instances trainSet, int maxNumIters, double lambda) {\n\t\tInstances trainSetNew = encoder.process(trainSet);\n\t\t\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setTask(Task.CLASSIFICATION);\n\t\tlearner.setLambda(lambda);\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setMaxNumIters(maxNumIters);\n\t\t\n\t\tGLM glm = learner.build(trainSetNew);\n\t\t\n\t\treturn GAMUtils.getGAM(glm, trainSet.getAttributes());\n\t}\n\t\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the L2 regularization parameter.\n\t * @return a regressor.\n\t */\n\tpublic GAM buildRegressor(Instances trainSet, int maxNumIters, double lambda) {\n\t\tInstances trainSetNew = encoder.process(trainSet);\n\t\t\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setTask(Task.REGRESSION);\n\t\tlearner.setLambda(lambda);\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setMaxNumIters(maxNumIters);\n\t\t\n\t\tGLM glm = learner.build(trainSetNew);\n\t\t\n\t\treturn GAMUtils.getGAM(glm, trainSet.getAttributes());\n\t}\n\n\t@Override\n\tpublic GAM build(Instances instances) {\n\t\tGAM gam = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = instances.dimension() * 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tgam = buildRegressor(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tgam = buildClassifier(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn gam;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/SparseDesignMatrix.java",
    "content": "package mltk.predictor.gam;\r\n\r\nimport java.util.HashSet;\r\nimport java.util.Set;\r\n\r\nimport mltk.predictor.function.CubicSpline;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.VectorUtils;\r\n\r\nclass SparseDesignMatrix {\r\n\r\n\tint[][] indices;\r\n\tdouble[][][] values;\r\n\tdouble[][] knots;\r\n\tdouble[][] std;\r\n\r\n\tSparseDesignMatrix(int[][] indices, double[][][] values, double[][] knots, double[][] std) {\r\n\t\tthis.indices = indices;\r\n\t\tthis.values = values;\r\n\t\tthis.knots = knots;\r\n\t\tthis.std = std;\r\n\t}\r\n\r\n\tstatic SparseDesignMatrix createCubicSplineDesignMatrix(int n, int[][] indices, double[][] values,\r\n\t\t\tdouble[] stdList, int numKnots) {\r\n\t\tfinal int p = indices.length;\r\n\t\tdouble[][][] x = new double[p][][];\r\n\t\tdouble[][] knots = new double[p][];\r\n\t\tdouble[][] std = new double[p][];\r\n\t\tdouble factor = Math.sqrt(n);\r\n\t\tfor (int j = 0; j < values.length; j++) {\r\n\t\t\tSet<Double> uniqueValues = new HashSet<>();\r\n\t\t\tdouble[] x1 = values[j];\r\n\t\t\tfor (int i = 0; i < values[j].length; i++) {\r\n\t\t\t\tuniqueValues.add(x1[i]);\r\n\t\t\t}\r\n\t\t\tint nKnots = uniqueValues.size() + 1 <= numKnots ? 0 : numKnots;\r\n\t\t\tknots[j] = new double[nKnots];\r\n\t\t\tif (nKnots != 0) {\r\n\t\t\t\tx[j] = new double[nKnots + 3][];\r\n\t\t\t\tstd[j] = new double[nKnots + 3];\r\n\t\t\t} else {\r\n\t\t\t\tx[j] = new double[1][];\r\n\t\t\t\tstd[j] = new double[1];\r\n\t\t\t}\r\n\t\t\tdouble[][] tX = x[j];\r\n\t\t\ttX[0] = x1;\r\n\t\t\tstd[j][0] = stdList[j] / factor;\r\n\t\t\tif (nKnots != 0) {\r\n\t\t\t\tdouble[] x2 = new double[x1.length];\r\n\t\t\t\tfor (int i = 0; i < x2.length; i++) {\r\n\t\t\t\t\tx2[i] = x1[i] * x1[i];\r\n\t\t\t\t}\r\n\t\t\t\ttX[1] = x2;\r\n\t\t\t\tdouble[] x3 = new double[x1.length];\r\n\t\t\t\tfor (int i = 0; i < x3.length; i++) {\r\n\t\t\t\t\tx3[i] = x2[i] * x1[i];\r\n\t\t\t\t}\r\n\t\t\t\ttX[2] = x3;\r\n\r\n\t\t\t\tstd[j][1] = StatUtils.sd(x2, n) / factor;\r\n\t\t\t\tstd[j][2] = StatUtils.sd(x3, n) / factor;\r\n\r\n\t\t\t\tdouble max = Math.max(StatUtils.max(x1), 0);\r\n\t\t\t\tdouble min = Math.min(StatUtils.min(x1), 0);\r\n\t\t\t\tdouble stepSize = (max - min) / nKnots;\r\n\t\t\t\tfor (int k = 0; k < nKnots; k++) {\r\n\t\t\t\t\tknots[j][k] = min + stepSize * k;\r\n\t\t\t\t\tdouble[] basis = new double[x1.length];\r\n\t\t\t\t\tdouble zero = CubicSpline.h(0, knots[j][k]);\r\n\t\t\t\t\tfor (int i = 0; i < basis.length; i++) {\r\n\t\t\t\t\t\tbasis[i] = CubicSpline.h(x1[i], knots[j][k]) - zero;\r\n\t\t\t\t\t}\r\n\t\t\t\t\tstd[j][k + 3] = StatUtils.sd(basis, n) / factor;\r\n\t\t\t\t\ttX[k + 3] = basis;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\t\r\n\t\t// Normalize the inputs\r\n\t\tfor (int j = 0; j < p; j++) {\r\n\t\t\tdouble[][] block = x[j];\r\n\t\t\tdouble[] s = std[j];\r\n\t\t\tfor (int i = 0; i < block.length; i++) {\r\n\t\t\t\tVectorUtils.divide(block[i], s[i]);\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn new SparseDesignMatrix(indices, x, knots, std);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/interaction/FAST.java",
    "content": "package mltk.predictor.gam.interaction;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.FileReader;\r\nimport java.io.PrintWriter;\r\nimport java.util.ArrayList;\r\nimport java.util.Collections;\r\nimport java.util.List;\r\n\r\nimport mltk.cmdline.Argument;\r\nimport mltk.cmdline.CmdLineParser;\r\nimport mltk.core.Attribute;\r\nimport mltk.core.BinnedAttribute;\r\nimport mltk.core.Instance;\r\nimport mltk.core.Instances;\r\nimport mltk.core.NominalAttribute;\r\nimport mltk.core.Attribute.Type;\r\nimport mltk.core.io.InstancesReader;\r\nimport mltk.core.processor.Discretizer;\r\nimport mltk.predictor.function.CHistogram;\r\nimport mltk.predictor.function.Histogram2D;\r\nimport mltk.util.Element;\r\nimport mltk.util.MathUtils;\r\nimport mltk.util.tuple.IntPair;\r\n\r\n/**\r\n * Class for fast interaction detection.\r\n * \r\n * <p>\r\n * Reference:<br>\r\n * Y. Lou, R. Caruana, J. Gehrke, and G. Hooker. Accurate intelligible models with pairwise interactions. In\r\n * <i>Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD)</i>,\r\n * Chicago, IL, USA, 2013.\r\n * </p>\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class FAST {\r\n\r\n\tstatic class FASTThread extends Thread {\r\n\r\n\t\tList<Element<IntPair>> pairs;\r\n\t\tInstances instances;\r\n\r\n\t\tFASTThread(Instances instances) {\r\n\t\t\tthis.instances = instances;\r\n\t\t\tthis.pairs = new ArrayList<>();\r\n\t\t}\r\n\r\n\t\tpublic void add(Element<IntPair> pair) {\r\n\t\t\tpairs.add(pair);\r\n\t\t}\r\n\r\n\t\tpublic void run() {\r\n\t\t\tFAST.computeWeights(instances, pairs);\r\n\t\t}\r\n\t}\r\n\r\n\tstatic class Options {\r\n\r\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\r\n\t\tString attPath = null;\r\n\r\n\t\t@Argument(name = \"-d\", description = \"dataset path\", required = true)\r\n\t\tString datasetPath = null;\r\n\r\n\t\t@Argument(name = \"-R\", description = \"residual path\", required = true)\r\n\t\tString residualPath = null;\r\n\r\n\t\t@Argument(name = \"-o\", description = \"output path\", required = true)\r\n\t\tString outputPath = null;\r\n\r\n\t\t@Argument(name = \"-b\", description = \"number of bins (default: 256)\")\r\n\t\tint maxNumBins = 256;\r\n\r\n\t\t@Argument(name = \"-p\", description = \"number of threads (default: 1)\")\r\n\t\tint numThreads = 1;\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Ranks pairwise interactions using FAST.\r\n\t * \r\n\t * <pre>\r\n\t * Usage: mltk.predictor.gam.interaction.FAST\r\n\t * -d\tdataset path\r\n\t * -R\tresidual path\r\n\t * -o\toutput path\r\n\t * [-r]\tattribute file path\r\n\t * [-b]\tnumber of bins (default: 256)\r\n\t * [-p]\tnumber of threads (default: 1)\r\n\t * </pre>\r\n\t * \r\n\t * @param args the command line arguments\r\n\t * @throws Exception\r\n\t */\r\n\tpublic static void main(String[] args) throws Exception {\r\n\t\tOptions opts = new Options();\r\n\t\tCmdLineParser parser = new CmdLineParser(FAST.class, opts);\r\n\t\ttry {\r\n\t\t\tparser.parse(args);\r\n\t\t} catch (IllegalArgumentException e) {\r\n\t\t\tparser.printUsage();\r\n\t\t\tSystem.exit(1);\r\n\t\t}\r\n\r\n\t\tInstances instances = InstancesReader.read(opts.attPath, opts.datasetPath);\r\n\r\n\t\tSystem.out.println(\"Reading residuals...\");\r\n\t\tBufferedReader br = new BufferedReader(new FileReader(opts.residualPath), 65535);\r\n\t\tfor (int i = 0; i < instances.size(); i++) {\r\n\t\t\tString line = br.readLine();\r\n\t\t\tdouble residual = Double.parseDouble(line);\r\n\t\t\tInstance instance = instances.get(i);\r\n\t\t\tinstance.setTarget(residual);\r\n\t\t}\r\n\t\tbr.close();\r\n\r\n\t\tList<Attribute> attributes = instances.getAttributes();\r\n\r\n\t\tSystem.out.println(\"Discretizing attribute...\");\r\n\t\tfor (int i = 0; i < attributes.size(); i++) {\r\n\t\t\tif (attributes.get(i).getType() == Type.NUMERIC) {\r\n\t\t\t\tDiscretizer.discretize(instances, i, opts.maxNumBins);\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tSystem.out.println(\"Generating all pairs of attributes...\");\r\n\t\tList<Element<IntPair>> pairs = new ArrayList<>();\r\n\t\tfor (int i = 0; i < attributes.size(); i++) {\r\n\t\t\tfor (int j = i + 1; j < attributes.size(); j++) {\r\n\t\t\t\tpairs.add(new Element<IntPair>(new IntPair(i, j), 0.0));\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tSystem.out.println(\"Creating threads...\");\r\n\t\tFASTThread[] threads = new FASTThread[opts.numThreads];\r\n\t\tlong start = System.currentTimeMillis();\r\n\t\tfor (int i = 0; i < threads.length; i++) {\r\n\t\t\tthreads[i] = new FASTThread(instances);\r\n\t\t}\r\n\t\tfor (int i = 0; i < pairs.size(); i++) {\r\n\t\t\tthreads[i % threads.length].add(pairs.get(i));\r\n\t\t}\r\n\t\tfor (int i = 0; i < threads.length; i++) {\r\n\t\t\tthreads[i].start();\r\n\t\t}\r\n\t\tSystem.out.println(\"Running FAST...\");\r\n\t\tfor (int i = 0; i < threads.length; i++) {\r\n\t\t\tthreads[i].join();\r\n\t\t}\r\n\t\tlong end = System.currentTimeMillis();\r\n\t\tSystem.out.println(\"Sorting pairs...\");\r\n\t\tCollections.sort(pairs);\r\n\r\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\r\n\r\n\t\tPrintWriter out = new PrintWriter(opts.outputPath);\r\n\t\tfor (int i = 0; i < pairs.size(); i++) {\r\n\t\t\tElement<IntPair> pair = pairs.get(i);\r\n\t\t\tout.println(pair.element.v1 + \"\\t\" + pair.element.v2 + \"\\t\" + pair.weight);\r\n\t\t}\r\n\t\tout.flush();\r\n\t\tout.close();\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Computes the weights of pairwise interactions.\r\n\t * \r\n\t * @param instances the training set.\r\n\t * @param pairs the list of pairs to compute.\r\n\t */\r\n\tpublic static void computeWeights(Instances instances, List<Element<IntPair>> pairs) {\r\n\t\tList<Attribute> attributes = instances.getAttributes();\r\n\t\tboolean[] used = new boolean[attributes.size()];\r\n\t\tfor (Element<IntPair> pair : pairs) {\r\n\t\t\tint f1 = pair.element.v1;\r\n\t\t\tint f2 = pair.element.v2;\r\n\t\t\tused[f1] = used[f2] = true;\r\n\t\t}\r\n\t\tCHistogram[] cHist = new CHistogram[attributes.size()];\r\n\t\tfor (int i = 0; i < cHist.length; i++) {\r\n\t\t\tif (used[i]) {\r\n\t\t\t\tswitch (attributes.get(i).getType()) {\r\n\t\t\t\t\tcase BINNED:\r\n\t\t\t\t\t\tBinnedAttribute binnedAtt = (BinnedAttribute) attributes.get(i);\r\n\t\t\t\t\t\tcHist[i] = new CHistogram(binnedAtt.getNumBins());\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t\tcase NOMINAL:\r\n\t\t\t\t\t\tNominalAttribute nominalAtt = (NominalAttribute) attributes.get(i);\r\n\t\t\t\t\t\tcHist[i] = new CHistogram(nominalAtt.getCardinality());\r\n\t\t\t\t\tdefault:\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\tdouble ySq = computeCHistograms(instances, used, cHist);\r\n\t\tfor (Element<IntPair> pair : pairs) {\r\n\t\t\tfinal int f1 = pair.element.v1;\r\n\t\t\tfinal int f2 = pair.element.v2;\r\n\t\t\tfinal int size1 = cHist[f1].size();\r\n\t\t\tfinal int size2 = cHist[f2].size();\r\n\t\t\tHistogram2D hist2d = new Histogram2D(size1, size2);\r\n\t\t\tHistogram2D.computeHistogram2D(instances, f1, f2, hist2d);\r\n\t\t\tcomputeWeight(pair, cHist, hist2d, ySq);\r\n\t\t}\r\n\t}\r\n\r\n\tprotected static double computeCHistograms(Instances instances, boolean[] used, CHistogram[] cHist) {\r\n\t\tdouble ySq = 0;\r\n\t\t// compute histogram\r\n\t\tfor (Instance instance : instances) {\r\n\t\t\tdouble resp = instance.getTarget();\r\n\t\t\tfor (int j = 0; j < instances.getAttributes().size(); j++) {\r\n\t\t\t\tif (used[j]) {\r\n\t\t\t\t\tif (!instance.isMissing(j)) {\r\n\t\t\t\t\t\tint idx = (int) instance.getValue(j);\r\n\t\t\t\t\t\tcHist[j].sum[idx] += resp * instance.getWeight();\r\n\t\t\t\t\t\tcHist[j].count[idx] += instance.getWeight();\r\n\t\t\t\t\t} else {\r\n\t\t\t\t\t\tcHist[j].sumOnMV += resp * instance.getWeight();\r\n\t\t\t\t\t\tcHist[j].countOnMV += instance.getWeight();\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tySq += resp * resp * instance.getWeight();\r\n\t\t}\r\n\t\t// compute cumulative histogram\r\n\t\tfor (int j = 0; j < cHist.length; j++) {\r\n\t\t\tif (used[j]) {\r\n\t\t\t\tfor (int idx = 1; idx < cHist[j].size(); idx++) {\r\n\t\t\t\t\tcHist[j].sum[idx] += cHist[j].sum[idx - 1];\r\n\t\t\t\t\tcHist[j].count[idx] += cHist[j].count[idx - 1];\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn ySq;\r\n\t}\r\n\r\n\tprotected static void computeWeight(Element<IntPair> pair, CHistogram[] cHist, Histogram2D hist2d, double ySq) {\r\n\t\tfinal int f1 = pair.element.v1;\r\n\t\tfinal int f2 = pair.element.v2;\r\n\t\tfinal int size1 = cHist[f1].size();\r\n\t\tfinal int size2 = cHist[f2].size();\r\n\t\tHistogram2D.Table table = Histogram2D.computeTable(hist2d, cHist[f1], cHist[f2]);\r\n\t\tdouble bestRSS = Double.POSITIVE_INFINITY;\r\n\t\tdouble[] predInt = new double[4];\r\n\t\tdouble[] predOnMV1 = new double[2];\r\n\t\tdouble[] predOnMV2 = new double[2];\r\n\t\tdouble predOnMV12 = MathUtils.divide(hist2d.respOnMV12, hist2d.countOnMV12, 0);\r\n\t\tfor (int v1 = 0; v1 < size1 - 1; v1++) {\r\n\t\t\tfor (int v2 = 0; v2 < size2 - 1; v2++) {\r\n\t\t\t\tgetPredictor(table, v1, v2, predInt, predOnMV1, predOnMV2);\r\n\t\t\t\tdouble rss = getRSS(table, v1, v2, ySq, predInt, predOnMV1, predOnMV2, predOnMV12);\r\n\t\t\t\tif (rss < bestRSS) {\r\n\t\t\t\t\tbestRSS = rss;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\tpair.weight = bestRSS;\r\n\t}\r\n\r\n\tprotected static void getPredictor(Histogram2D.Table table, int v1, int v2,\r\n\t\t\tdouble[] pred, double[] predOnMV1, double[] predOnMV2) {\r\n\t\tdouble[] count = table.count[v1][v2];\r\n\t\tdouble[] resp = table.resp[v1][v2];\r\n\t\tfor (int i = 0; i < pred.length; i++) {\r\n\t\t\tpred[i] = MathUtils.divide(resp[i], count[i], 0);\r\n\t\t}\r\n\t\tfor (int i = 0; i < predOnMV1.length; i++) {\r\n\t\t\tpredOnMV1[i] = MathUtils.divide(table.respOnMV1[v2][i], table.countOnMV1[v2][i], 0);\r\n\t\t}\r\n\t\tfor (int i = 0; i < predOnMV2.length; i++) {\r\n\t\t\tpredOnMV2[i] = MathUtils. divide(table.respOnMV2[v1][i], table.countOnMV2[v1][i], 0);\r\n\t\t}\r\n\t}\r\n\r\n\tprotected static double getRSS(Histogram2D.Table table, int v1, int v2, double ySq,\r\n\t\t\tdouble[] pred, double[] predOnMV1, double[] predOnMV2, double predOnMV12) {\r\n\t\tdouble[] count = table.count[v1][v2];\r\n\t\tdouble[] resp = table.resp[v1][v2];\r\n\t\tdouble[] respOnMV1 = table.respOnMV1[v2];\r\n\t\tdouble[] countOnMV1 = table.countOnMV1[v2];\r\n\t\tdouble[] respOnMV2 = table.respOnMV2[v1];\r\n\t\tdouble[] countOnMV2 = table.countOnMV2[v1];\r\n\t\tdouble rss = ySq;\r\n\t\t// Compute main area\r\n\t\tdouble t = 0;\r\n\t\tfor (int i = 0; i < pred.length; i++) {\r\n\t\t\tt += pred[i] * pred[i] * count[i];\r\n\t\t}\r\n\t\trss += t;\r\n\t\tt = 0;\r\n\t\tfor (int i = 0; i < pred.length; i++) {\r\n\t\t\tt += pred[i] * resp[i];\r\n\t\t}\r\n\t\trss -= 2 * t;\r\n\t\t// Compute on mv1\r\n\t\tt = 0;\r\n\t\tfor (int i = 0; i < predOnMV1.length; i++) {\r\n\t\t\tt += predOnMV1[i] * predOnMV1[i] * countOnMV1[i];\r\n\t\t}\r\n\t\trss += t;\r\n\t\tt = 0;\r\n\t\tfor (int i = 0; i < predOnMV1.length; i++) {\r\n\t\t\tt += predOnMV1[i] * respOnMV1[i];\r\n\t\t}\r\n\t\trss -= 2 * t;\r\n\t\t// Compute on mv2\r\n\t\tt = 0;\r\n\t\tfor (int i = 0; i < predOnMV2.length; i++) {\r\n\t\t\tt += predOnMV2[i] * predOnMV2[i] * countOnMV2[i];\r\n\t\t}\r\n\t\trss += t;\r\n\t\tt = 0;\r\n\t\tfor (int i = 0; i < predOnMV2.length; i++) {\r\n\t\t\tt += predOnMV2[i] * respOnMV2[i];\r\n\t\t}\r\n\t\trss -= 2 * t;\r\n\t\treturn rss;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/interaction/package-info.java",
    "content": "/**\n * Provides algorithms for feature interaction detection.\n */\npackage mltk.predictor.gam.interaction;"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/package-info.java",
    "content": "/**\n * Provides algorithms for fitting generalized additive models (GAMs).\n */\npackage mltk.predictor.gam;"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/tool/Diagnostics.java",
    "content": "package mltk.predictor.gam.tool;\n\nimport java.io.PrintWriter;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.gam.GAM;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.StatUtils;\nimport mltk.util.Element;\n\n/**\n * Class for GAM diagnostics.\n * \n * @author Yin Lou\n * \n */\npublic class Diagnostics {\n\t\n\t/**\n\t * Enumeration of methods for calculating term importance.\n\t * \n\t * @author Yin Lou\n\t * \n\t */\n\tpublic enum Mode {\n\n\t\t/**\n\t\t * L1.\n\t\t */\n\t\tL1(\"L1\"),\n\t\t/**\n\t\t * PDF terminal.\n\t\t */\n\t\tL2(\"L2\");\n\n\t\tString mode;\n\n\t\tMode(String mode) {\n\t\t\tthis.mode = mode;\n\t\t}\n\n\t\tpublic String toString() {\n\t\t\treturn mode;\n\t\t}\n\n\t\t/**\n\t\t * Parses a mode from a string.\n\t\t * \n\t\t * @param mode the mode.\n\t\t * @return a parsed terminal.\n\t\t */\n\t\tpublic static Mode getEnum(String mode) {\n\t\t\tfor (Mode re : Mode.values()) {\n\t\t\t\tif (re.mode.compareTo(mode) == 0) {\n\t\t\t\t\treturn re;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new IllegalArgumentException(\"Invalid mode: \" + mode);\n\t\t}\n\n\t}\n\n\t/**\n\t * Computes the weights for each term in a GAM.\n\t * \n\t * @param gam the GAM model.\n\t * @param instances the training set.\n\t * @return the list of weights for each term in a GAM.\n\t */\n\tpublic static List<Element<int[]>> diagnose(GAM gam, Instances instances) {\n\t\treturn diagnose(gam, instances, Mode.L2);\n\t}\n\t\n\t/**\n\t * Computes the weights for each term in a GAM.\n\t * \n\t * @param gam the GAM model.\n\t * @param instances the training set.\n\t * @return the list of weights for each term in a GAM.\n\t */\n\tpublic static List<Element<int[]>> diagnose(GAM gam, Instances instances, Mode mode) {\n\t\tList<Element<int[]>> list = new ArrayList<>();\n\t\tMap<int[], List<Regressor>> map = new HashMap<>();\n\t\tList<int[]> terms = gam.getTerms();\n\t\tList<Regressor> regressors = gam.getRegressors();\n\t\tfor (int i = 0; i < terms.size(); i++) {\n\t\t\tint[] term = terms.get(i);\n\t\t\tif (!map.containsKey(term)) {\n\t\t\t\tmap.put(term, new ArrayList<Regressor>());\n\t\t\t}\n\t\t\tRegressor regressor = regressors.get(i);\n\t\t\tmap.get(term).add(regressor);\n\t\t}\n\n\t\tdouble[] predictions = new double[instances.size()];\n\t\tfor (int[] term : map.keySet()) {\n\t\t\tList<Regressor> regressorList = map.get(term);\n\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\tpredictions[i] = 0;\n\t\t\t\tInstance instance = instances.get(i);\n\t\t\t\tfor (Regressor regressor : regressorList) {\n\t\t\t\t\tpredictions[i] += regressor.regress(instance);\n\t\t\t\t}\n\t\t\t}\n\t\t\tdouble weight = 0;\n\t\t\tif (mode == Mode.L2) {\n\t\t\t\tweight = StatUtils.variance(predictions);\n\t\t\t} else {\n\t\t\t\tdouble mean = StatUtils.mean(predictions);\n\t\t\t\tweight = StatUtils.mad(predictions, mean);\n\t\t\t}\n\t\t\t\n\t\t\tlist.add(new Element<int[]>(term, weight));\n\t\t}\n\n\t\treturn list;\n\t}\n\n\tstatic class Options {\n\n\t\t@Argument(name = \"-r\", description = \"attribute file path\")\n\t\tString attPath = null;\n\n\t\t@Argument(name = \"-d\", description = \"dataset path\", required = true)\n\t\tString datasetPath = null;\n\t\t\n\t\t@Argument(name = \"-m\", description = \"mode (L1 or L2, default: L2)\")\n\t\tString mode = null;\n\n\t\t@Argument(name = \"-i\", description = \"input model path\", required = true)\n\t\tString inputModelPath = null;\n\n\t\t@Argument(name = \"-o\", description = \"output path\", required = true)\n\t\tString outputPath = null;\n\n\t}\n\n\t/**\n\t * Generates term importance for GAMs.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.tool.Diagnostics\n\t * -d\tdataset path\n\t * -i\tinput model path\n\t * -o\toutput path\n\t * [-r]\tattribute file path\n\t * [-m]\tmode (L1 or L2, default: L2)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(Diagnostics.class, opts);\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\t\tInstances dataset = InstancesReader.read(opts.attPath, opts.datasetPath);\n\t\tGAM gam = PredictorReader.read(opts.inputModelPath, GAM.class);\n\n\t\tList<Element<int[]>> list = Diagnostics.diagnose(gam, dataset, Mode.getEnum(opts.mode));\n\t\tCollections.sort(list);\n\t\tCollections.reverse(list);\n\n\t\tPrintWriter out = new PrintWriter(opts.outputPath);\n\t\tfor (Element<int[]> element : list) {\n\t\t\tint[] term = element.element;\n\t\t\tdouble weight = element.weight;\n\t\t\tout.println(Arrays.toString(term) + \": \" + weight);\n\t\t}\n\t\tout.flush();\n\t\tout.close();\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/tool/Visualizer.java",
    "content": "package mltk.predictor.gam.tool;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.PrintWriter;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.core.Attribute;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Bins;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.function.Array1D;\nimport mltk.predictor.function.Array2D;\nimport mltk.predictor.function.CubicSpline;\nimport mltk.predictor.function.Function1D;\nimport mltk.predictor.gam.GAM;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.MathUtils;\n\n/**\n * Class for visualizing 1D and 2D components in a GAM.\n * \n * @author Yin Lou\n * \n */\npublic class Visualizer {\n\n\t/**\n\t * Enumeration of output terminals.\n\t * \n\t * @author Yin Lou\n\t * \n\t */\n\tpublic enum Terminal {\n\n\t\t/**\n\t\t * PNG terminal.\n\t\t */\n\t\tPNG(\"png\"),\n\t\t/**\n\t\t * PDF terminal.\n\t\t */\n\t\tPDF(\"pdf\");\n\n\t\tString term;\n\n\t\tTerminal(String term) {\n\t\t\tthis.term = term;\n\t\t}\n\n\t\tpublic String toString() {\n\t\t\treturn term;\n\t\t}\n\n\t\t/**\n\t\t * Parses an enumeration from a string.\n\t\t * \n\t\t * @param term the string.\n\t\t * @return a parsed terminal.\n\t\t */\n\t\tpublic static Terminal getEnum(String term) {\n\t\t\tfor (Terminal re : Terminal.values()) {\n\t\t\t\tif (re.term.compareTo(term) == 0) {\n\t\t\t\t\treturn re;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new IllegalArgumentException(\"Invalid Terminal value: \" + term);\n\t\t}\n\n\t}\n\n\t/**\n\t * Generates a set of Gnuplot scripts for visualizing low dimensional components in a GAM.\n\t * \n\t * @param gam the GAM model.\n\t * @param instances the training set.\n\t * @param dirPath the directory path to write to.\n\t * @param outputTerminal output plot format (png or pdf).\n\t * @throws IOException\n\t */\n\tpublic static void generateGnuplotScripts(GAM gam, Instances instances, String dirPath, Terminal outputTerminal)\n\t\t\tthrows IOException {\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tint p = -1;\n\t\tMap<Integer, Attribute> attMap = new HashMap<>(attributes.size());\n\t\tfor (Attribute attribute : attributes) {\n\t\t\tint attIndex = attribute.getIndex();\n\t\t\tattMap.put(attIndex, attribute);\n\t\t\tif (attIndex > p) {\n\t\t\t\tp = attIndex;\n\t\t\t}\n\t\t}\n\t\tp++;\n\t\t\n\t\tList<int[]> terms = gam.getTerms();\n\t\tList<Regressor> regressors = gam.getRegressors();\n\n\t\tFile dir = new File(dirPath);\n\t\tif (!dir.exists()) {\n\t\t\tdir.mkdirs();\n\t\t}\n\n\t\tdouble[] value = new double[p];\n\t\tInstance point = new Instance(value);\n\n\t\tString terminal = outputTerminal.toString();\n\t\tfor (int i = 0; i < terms.size(); i++) {\n\t\t\tint[] term = terms.get(i);\n\t\t\tRegressor regressor = regressors.get(i);\n\t\t\tif (term.length == 1) {\n\t\t\t\tAttribute f = attMap.get(term[0]);\n\t\t\t\tswitch (f.getType()) {\n\t\t\t\t\tcase BINNED:\n\t\t\t\t\t\tint numBins = ((BinnedAttribute) f).getNumBins();\n\t\t\t\t\t\tif (numBins == 1) {\n\t\t\t\t\t\t\tcontinue;\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tcase NOMINAL:\n\t\t\t\t\t\tint numStates = ((NominalAttribute) f).getStates().length;\n\t\t\t\t\t\tif (numStates == 1) {\n\t\t\t\t\t\t\tcontinue;\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tDouble predictionOnMV = null;\n\t\t\t\tif (regressor instanceof Function1D) {\n\t\t\t\t\tdouble predOnMV = ((Function1D) regressor).getPredictionOnMV();\n\t\t\t\t\tif (!MathUtils.isZero(predOnMV)) {\n\t\t\t\t\t\tpredictionOnMV = predOnMV;\n\t\t\t\t\t}\n\t\t\t\t} else if (regressor instanceof Array1D) {\n\t\t\t\t\tdouble predOnMV = ((Array1D) regressor).getPredictionOnMV();\n\t\t\t\t\tif (!MathUtils.isZero(predOnMV)) {\n\t\t\t\t\t\tpredictionOnMV = predOnMV;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tPrintWriter out = new PrintWriter(dir.getAbsolutePath() + File.separator + f.getName() + \".plt\");\n\t\t\t\tout.printf(\"set term %s\\n\", terminal);\n\t\t\t\tout.printf(\"set output \\\"%s.%s\\\"\\n\", f.getName(), terminal);\n\t\t\t\tout.println(\"set datafile separator \\\"\\t\\\"\");\n\t\t\t\tout.println(\"set grid\");\n\t\t\t\tif (predictionOnMV != null) {\n\t\t\t\t\tout.println(\"set multiplot layout 1,2 rowsfirst\");\n\t\t\t\t}\n\t\t\t\t// Plot main function\n\t\t\t\tswitch (f.getType()) {\n\t\t\t\t\tcase BINNED:\n\t\t\t\t\t\tint numBins = ((BinnedAttribute) f).getNumBins();\n\t\t\t\t\t\tBins bins = ((BinnedAttribute) f).getBins();\n\t\t\t\t\t\tdouble[] boundaries = bins.getBoundaries();\n\t\t\t\t\t\tdouble start = boundaries[0] - 1;\n\t\t\t\t\t\tif (boundaries.length >= 2) {\n\t\t\t\t\t\t\tstart = boundaries[0] - (boundaries[1] - boundaries[0]);\n\t\t\t\t\t\t}\n\t\t\t\t\t\tout.printf(\"set xrange[%f:%f]\\n\", start, boundaries[boundaries.length - 1]);\n\t\t\t\t\t\tList<Double> predList = new ArrayList<>();\n\t\t\t\t\t\tfor (int j = 0; j < numBins; j++) {\n\t\t\t\t\t\t\tpoint.setValue(term[0], j);\n\t\t\t\t\t\t\tpredList.add(regressor.regress(point));\n\t\t\t\t\t\t}\n\t\t\t\t\t\tpoint.setValue(term[0], 0);\n\t\t\t\t\t\t{// Writing plot data to file\n\t\t\t\t\t\t\tString fileName = f.getName() + \".dat\";\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"\\\" u 1:2 w l t \\\"\\\"\");\n\t\t\t\t\t\t\tPrintWriter writer = new PrintWriter(dir.getAbsolutePath() + File.separator + fileName);\n\t\t\t\t\t\t\twriter.printf(\"%f\\t%f\\n\", start, predList.get(0));\n\t\t\t\t\t\t\tfor (int j = 0; j < numBins; j++) {\n\t\t\t\t\t\t\t\tpoint.setValue(term[0], j);\n\t\t\t\t\t\t\t\twriter.printf(\"%f\\t%f\\n\", boundaries[j], predList.get(j));\n\t\t\t\t\t\t\t\tif (j < numBins - 1) {\n\t\t\t\t\t\t\t\t\twriter.printf(\"%f\\t%f\\n\", boundaries[j], predList.get(j + 1));\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\twriter.flush();\n\t\t\t\t\t\t\twriter.close();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tcase NOMINAL:\n\t\t\t\t\t\tout.println(\"set style data histogram\");\n\t\t\t\t\t\tout.println(\"set style histogram cluster gap 1\");\n\t\t\t\t\t\tout.println(\"set style fill solid border -1\");\n\t\t\t\t\t\tout.println(\"set boxwidth 0.9\");\n\t\t\t\t\t\tout.println(\"set xtic rotate by -90\");\n\t\t\t\t\t\tString[] states = ((NominalAttribute) f).getStates();\n\t\t\t\t\t\t{// Writing plot data to file\n\t\t\t\t\t\t\tString fileName = f.getName() + \".dat\";\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"\\\" u 2:xtic(1) t \\\"\\\"\");\n\t\t\t\t\t\t\tPrintWriter writer = new PrintWriter(dir.getAbsolutePath() + File.separator + fileName);\n\t\t\t\t\t\t\tfor (int j = 0; j < states.length; j++) {\n\t\t\t\t\t\t\t\tpoint.setValue(term[0], j);\n\t\t\t\t\t\t\t\twriter.printf(\"%s\\t%f\\n\", states[j], regressor.regress(point));\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\twriter.flush();\n\t\t\t\t\t\t\twriter.close();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tSet<Double> values = new HashSet<>();\n\t\t\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\t\t\tvalues.add(instance.getValue(term[0]));\n\t\t\t\t\t\t}\n\t\t\t\t\t\tList<Double> list = new ArrayList<>(values);\n\t\t\t\t\t\tCollections.sort(list);\n\t\t\t\t\t\tout.printf(\"set xrange[%f:%f]\\n\", list.get(0), list.get(list.size() - 1));\n\t\t\t\t\t\tif (regressor instanceof CubicSpline) {\n\t\t\t\t\t\t\tCubicSpline spline = (CubicSpline) regressor;\n\t\t\t\t\t\t\tout.println(\"z(x) = x < 0 ? 0 : x ** 3\");\n\t\t\t\t\t\t\tout.println(\"h(x, k) = z(x - k)\");\n\t\t\t\t\t\t\tdouble[] knots = spline.getKnots();\n\t\t\t\t\t\t\tdouble[] w = spline.getCoefficients();\n\t\t\t\t\t\t\tStringBuilder sb = new StringBuilder();\n\t\t\t\t\t\t\tsb.append(\"plot \").append(spline.getIntercept());\n\t\t\t\t\t\t\tsb.append(\" + \").append(w[0]).append(\" * x\");\n\t\t\t\t\t\t\tsb.append(\" + \").append(w[1]).append(\" * (x ** 2)\");\n\t\t\t\t\t\t\tsb.append(\" + \").append(w[2]).append(\" * (x ** 3)\");\n\t\t\t\t\t\t\tfor (int j = 0; j < knots.length; j++) {\n\t\t\t\t\t\t\t\tsb.append(\" + \").append(w[j + 3]).append(\" * \");\n\t\t\t\t\t\t\t\tsb.append(\"h(x, \").append(knots[j]).append(\")\");\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tsb.append(\" t \\\"\\\"\");\n\t\t\t\t\t\t\tout.println(sb.toString());\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tout.println(\"plot \\\"-\\\" u 1:2 w lp t \\\"\\\"\");\n\t\t\t\t\t\t\tfor (double v : list) {\n\t\t\t\t\t\t\t\tpoint.setValue(term[0], v);\n\t\t\t\t\t\t\t\tout.printf(\"%f\\t%f\\n\", v, regressor.regress(point));\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tout.println(\"e\");\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\t// Plot prediction on missing value\n\t\t\t\tif (predictionOnMV != null) {\n\t\t\t\t\tout.println(\"set style fill solid border -1\");\n\t\t\t\t\tout.println(\"set xtic rotate by 0\");\n\t\t\t\t\tout.println(\"plot \\\"-\\\" using 2:xtic(1) with histogram t \\\"\\\"\");\n\t\t\t\t\tout.println(\"missing value\\t\" + predictionOnMV);\n\t\t\t\t\tout.println(\"e\");\n\t\t\t\t}\n\t\t\t\tout.flush();\n\t\t\t\tout.close();\n\t\t\t} else if (term.length == 2) {\n\t\t\t\tAttribute f1 = attMap.get(term[0]);\n\t\t\t\tAttribute f2 = attMap.get(term[1]);\n\t\t\t\tString fileName = f1.getName() + \"_\" + f2.getName();\n\t\t\t\tPrintWriter out = new PrintWriter(dir.getAbsolutePath() \n\t\t\t\t\t\t+ File.separator + fileName + \".plt\");\n\t\t\t\tout.printf(\"set term %s\\n\", terminal);\n\t\t\t\tout.printf(\"set output \\\"%s_%s.%s\\\"\\n\", f1.getName(), \n\t\t\t\t\t\tf2.getName(), terminal);\n\t\t\t\tout.println(\"set datafile separator \\\"\\t\\\"\");\n\t\t\t\tint numRow = 1;\n\t\t\t\tint numCol = 1;\n\t\t\t\tdouble[] predictionsOnMV1 = null;\n\t\t\t\tdouble[] predictionsOnMV2 = null;\n\t\t\t\tdouble predictionsOnMV12 = 0.0;\n\t\t\t\tif (regressor instanceof Array2D) {\n\t\t\t\t\tArray2D ary2d = (Array2D) regressor;\n\t\t\t\t\tfor (double v : ary2d.getPredictionsOnMV1()) {\n\t\t\t\t\t\tif (!MathUtils.isZero(v)) {\n\t\t\t\t\t\t\tnumRow = 2;\n\t\t\t\t\t\t\tpredictionsOnMV1 = ary2d.getPredictionsOnMV1();\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tfor (double v : ary2d.getPredictionsOnMV2()) {\n\t\t\t\t\t\tif (!MathUtils.isZero(v)) {\n\t\t\t\t\t\t\tnumCol = 2;\n\t\t\t\t\t\t\tpredictionsOnMV2 = ary2d.getPredictionsOnMV2();\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif (!MathUtils.isZero(ary2d.getPredictionOnMV12())) {\n\t\t\t\t\t\tnumRow = 2;\n\t\t\t\t\t\tnumCol = 2;\n\t\t\t\t\t\tpredictionsOnMV12 = ary2d.getPredictionOnMV12();\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif (numRow > 1 || numCol > 1) {\n\t\t\t\t\tout.printf(\"set multiplot layout %d,%d rowsfirst\\n\", numRow, numCol);\n\t\t\t\t}\n\t\t\t\tint size1 = 0;\n\t\t\t\tif (f1.getType() == Attribute.Type.BINNED) {\n\t\t\t\t\tsize1 = ((BinnedAttribute) f1).getNumBins();\n\t\t\t\t} else if (f1.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\tsize1 = ((NominalAttribute) f1).getCardinality();\n\t\t\t\t}\n\t\t\t\tint size2 = 0;\n\t\t\t\tif (f2.getType() == Attribute.Type.BINNED) {\n\t\t\t\t\tsize2 = ((BinnedAttribute) f2).getNumBins();\n\t\t\t\t} else if (f2.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\tsize2 = ((NominalAttribute) f2).getCardinality();\n\t\t\t\t}\n\t\t\t\tif (f1.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\tout.print(\"set ytics(\");\n\t\t\t\t\tString[] states = ((NominalAttribute) f1).getStates();\n\t\t\t\t\tfor (int j = 0; j < states.length - 1; j++) {\n\t\t\t\t\t\tout.printf(\"\\\"%s\\\" %d, \", states[j], j);\n\t\t\t\t\t}\n\t\t\t\t\tout.printf(\"\\\"%s\\\" %d)\\n\", states[states.length - 1], states.length - 1);\n\t\t\t\t}\n\t\t\t\tif (f2.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\tout.print(\"set xtics(\");\n\t\t\t\t\tString[] states = ((NominalAttribute) f2).getStates();\n\t\t\t\t\tfor (int j = 0; j < states.length - 1; j++) {\n\t\t\t\t\t\tout.printf(\"\\\"%s\\\" %d, \", states[j], j);\n\t\t\t\t\t}\n\t\t\t\t\tout.printf(\"\\\"%s\\\" %d) rotate\\n\", states[states.length - 1], states.length - 1);\n\t\t\t\t}\n\t\t\t\tout.println(\"unset border\");\n\t\t\t\tout.println(\"set view map\");\n\t\t\t\tout.println(\"set style data pm3d\");\n\t\t\t\tout.println(\"set style function pm3d\");\n\t\t\t\tout.println(\"set pm3d corners2color c4\");\n\t\t\t\tdouble[] rangeY = null;\n\t\t\t\tdouble[] valueY = null;\n\t\t\t\tdouble[] rangeX = null;\n\t\t\t\tdouble[] valueX = null;\n\t\t\t\tif (f1 instanceof BinnedAttribute) {\n\t\t\t\t\trangeY = new double[size1 + 1];\n\t\t\t\t\tvalueY = new double[size1 + 1];\n\t\t\t\t\tBins bins = ((BinnedAttribute) f1).getBins();\n\t\t\t\t\tdouble[] boundaries = bins.getBoundaries();\n\t\t\t\t\tdouble start = boundaries[0] - 1;\n\t\t\t\t\tif (boundaries.length >= 2) {\n\t\t\t\t\t\tstart = boundaries[0] - (boundaries[1] - boundaries[0]);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\trangeY[0] = start;\n\t\t\t\t\tvalueY[0] = 0;\n\t\t\t\t\tfor (int k = 0; k < boundaries.length; k++) {\n\t\t\t\t\t\trangeY[k + 1] = boundaries[k];\n\t\t\t\t\t\tvalueY[k + 1] = k;\n\t\t\t\t\t}\n\t\t\t\t} else if (f1 instanceof NominalAttribute) {\n\t\t\t\t\trangeY = new double[size1];\n\t\t\t\t\tvalueY = new double[size1];\n\t\t\t\t\tfor (int k = 0; k < size1; k++) {\n\t\t\t\t\t\trangeY[k] = k;\n\t\t\t\t\t\tvalueY[k] = k;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tout.printf(\"set yrange[%f:%f]\\n\", rangeY[0] - 1, rangeY[rangeY.length - 1] + 1);\n\t\t\t\tif (f2 instanceof BinnedAttribute) {\n\t\t\t\t\trangeX = new double[size2 + 1];\n\t\t\t\t\tvalueX = new double[size2 + 1];\n\t\t\t\t\tBins bins = ((BinnedAttribute) f2).getBins();\n\t\t\t\t\tdouble[] boundaries = bins.getBoundaries();\n\t\t\t\t\tdouble start = boundaries[0] - 1;\n\t\t\t\t\tif (boundaries.length >= 2) {\n\t\t\t\t\t\tstart = boundaries[0] - (boundaries[1] - boundaries[0]);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\trangeX[0] = start;\n\t\t\t\t\tvalueX[0] = 0;\n\t\t\t\t\tfor (int k = 0; k < boundaries.length; k++) {\n\t\t\t\t\t\trangeX[k + 1] = boundaries[k];\n\t\t\t\t\t\tvalueX[k + 1] = k;\n\t\t\t\t\t}\n\t\t\t\t} else if (f2 instanceof NominalAttribute) {\n\t\t\t\t\trangeX = new double[size2];\n\t\t\t\t\tvalueX = new double[size2];\n\t\t\t\t\tfor (int k = 0; k < size2; k++) {\n\t\t\t\t\t\trangeX[k] = k;\n\t\t\t\t\t\tvalueX[k] = k;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tout.printf(\"set xrange[%f:%f]\\n\", rangeX[0] - 1, rangeX[rangeX.length - 1] + 1);\n\t\t\t\t\n\t\t\t\tout.println(\"splot \\\"\" + fileName + \".dat\\\" with image t \\\"\\\"\");\n\t\t\t\tPrintWriter writer = new PrintWriter(dir.getAbsolutePath() + File.separator + fileName + \".dat\");\n\t\t\t\tfor (int r = 0; r < rangeY.length; r++) {\n\t\t\t\t\tpoint.setValue(term[0], valueY[r]);\n\t\t\t\t\tfor (int c = 0; c < rangeX.length; c++) {\n\t\t\t\t\t\tpoint.setValue(term[1], valueX[c]);\n\t\t\t\t\t\twriter.println(rangeX[c] + \"\\t\" + rangeY[r] + \"\\t\" + gam.regress(point));\n\t\t\t\t\t}\n\t\t\t\t\twriter.println();\n\t\t\t\t}\n\t\t\t\twriter.flush();\n\t\t\t\twriter.close();\n\t\t\t\t\n\t\t\t\tif (numRow > 1 || numCol > 1) {\n\t\t\t\t\t// multiplot is on\n\t\t\t\t\tif (predictionsOnMV2 != null) {\n\t\t\t\t\t\tout.println(\"reset\");\n\t\t\t\t\t\twriter = new PrintWriter(dir.getAbsolutePath() + File.separator + fileName + \"_mv2.dat\");\n\t\t\t\t\t\tif (f1.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\t\t\tString[] states = ((NominalAttribute) f1).getStates();\n\t\t\t\t\t\t\tout.println(\"set style data histogram\");\n\t\t\t\t\t\t\tout.println(\"set style histogram cluster gap 1\");\n\t\t\t\t\t\t\tout.println(\"set style fill solid border -1\");\n\t\t\t\t\t\t\tout.println(\"set boxwidth 0.9\");\n\t\t\t\t\t\t\tout.println(\"set xtic rotate by -90\");\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"_mv2.dat\\\" u 2:xtic(1) t \\\"\\\"\");\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tfor (int k = 0; k < states.length; k++) {\n\t\t\t\t\t\t\t\twriter.println(states[k] + \"\\t\" + predictionsOnMV2[k]);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else if (f1.getType() == Attribute.Type.BINNED) {\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"_mv2.dat\\\" u 1:2 w l t \\\"\\\"\");\n\t\t\t\t\t\t\twriter.println(rangeY[0] + \"\\t\" + predictionsOnMV2[0]);\n\t\t\t\t\t\t\tfor (int k = 0; k < predictionsOnMV2.length; k++) {\n\t\t\t\t\t\t\t\twriter.println(rangeY[k + 1] + \"\\t\" + predictionsOnMV2[k]);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.flush();\n\t\t\t\t\t\twriter.close();\n\t\t\t\t\t} else if (numCol > 1) {\n\t\t\t\t\t\tout.println(\"set multiplot next\");\n\t\t\t\t\t}\n\t\t\t\t\tif (predictionsOnMV1 != null) {\n\t\t\t\t\t\tout.println(\"reset\");\n\t\t\t\t\t\twriter = new PrintWriter(dir.getAbsolutePath() + File.separator + fileName + \"_mv1.dat\");\n\t\t\t\t\t\tif (f2.getType() == Attribute.Type.NOMINAL) {\n\t\t\t\t\t\t\tString[] states = ((NominalAttribute) f2).getStates();\n\t\t\t\t\t\t\tout.println(\"set style data histogram\");\n\t\t\t\t\t\t\tout.println(\"set style histogram cluster gap 1\");\n\t\t\t\t\t\t\tout.println(\"set style fill solid border -1\");\n\t\t\t\t\t\t\tout.println(\"set boxwidth 0.9\");\n\t\t\t\t\t\t\tout.println(\"set xtic rotate by -90\");\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"_mv1.dat\\\" u 2:xtic(1) t \\\"\\\"\");\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tfor (int k = 0; k < states.length; k++) {\n\t\t\t\t\t\t\t\twriter.println(states[k] + \"\\t\" + predictionsOnMV1[k]);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else if (f2.getType() == Attribute.Type.BINNED) {\n\t\t\t\t\t\t\tout.println(\"plot \\\"\" + fileName + \"_mv1.dat\\\" u 1:2 w l t \\\"\\\"\");\n\t\t\t\t\t\t\twriter.println(rangeX[0] + \"\\t\" + predictionsOnMV1[0]);\n\t\t\t\t\t\t\tfor (int k = 0; k < predictionsOnMV1.length; k++) {\n\t\t\t\t\t\t\t\twriter.println(rangeX[k + 1] + \"\\t\" + predictionsOnMV1[k]);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.flush();\n\t\t\t\t\t\twriter.close();\n\t\t\t\t\t} else if (numRow > 1) {\n\t\t\t\t\t\tout.println(\"set multiplot next\");\n\t\t\t\t\t}\n\t\t\t\t\tif (!MathUtils.isZero(predictionsOnMV12)) {\n\t\t\t\t\t\tout.println(\"reset\");\n\t\t\t\t\t\tout.println(\"set datafile separator \\\"\\t\\\"\");\n\t\t\t\t\t\tout.println(\"set style fill solid border -1\");\n\t\t\t\t\t\tout.println(\"set xtic rotate by 0\");\n\t\t\t\t\t\tout.println(\"plot \\\"-\\\" using 2:xtic(1) with histogram t \\\"\\\"\");\n\t\t\t\t\t\tout.println(\"missing value\\t\" + predictionsOnMV12);\n\t\t\t\t\t\tout.println(\"e\");\n\t\t\t\t\t} else if (numRow > 1 && numCol > 1) {\n\t\t\t\t\t\tout.println(\"set multiplot next\");\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\t\n\t\t\t\tout.flush();\n\t\t\t\tout.close();\n\t\t\t}\n\t\t}\n\t}\n\n\tstatic class Options {\n\n\t\t@Argument(name = \"-r\", description = \"attribute file path\", required = true)\n\t\tString attPath = null;\n\n\t\t@Argument(name = \"-d\", description = \"dataset path\", required = true)\n\t\tString datasetPath = null;\n\n\t\t@Argument(name = \"-i\", description = \"input model path\", required = true)\n\t\tString inputModelPath = null;\n\n\t\t@Argument(name = \"-o\", description = \"output directory path\", required = true)\n\t\tString dirPath = null;\n\n\t\t@Argument(name = \"-t\", description = \"output terminal (default: png)\")\n\t\tString terminal = \"png\";\n\n\t}\n\n\t/**\n\t * Generates scripts for visualizing GAMs.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.gam.tool.Visualizer\n\t * -r\tattribute file path\n\t * -d\tdataset path\n\t * -i\tinput model path\n\t * -o\toutput directory path\n\t * [-t]\toutput terminal (default: png)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(Visualizer.class, opts);\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\t\tInstances dataset = InstancesReader.read(opts.attPath, opts.datasetPath);\n\t\tGAM gam = PredictorReader.read(opts.inputModelPath, GAM.class);\n\n\t\tVisualizer.generateGnuplotScripts(gam, dataset, opts.dirPath, Terminal.getEnum(opts.terminal));\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/gam/tool/package-info.java",
    "content": "/**\n * Provides tools for diagnosing and visualizing GAMs.\n */\npackage mltk.predictor.gam.tool;"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/ElasticNetLearner.java",
    "content": "package mltk.predictor.glm;\n\nimport java.util.Arrays;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerWithTaskOptions;\nimport mltk.core.Attribute;\nimport mltk.core.DenseVector;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.SparseVector;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Family;\nimport mltk.predictor.LinkFunction;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for learning elastic-net penalized linear model via coordinate descent.\n * \n * @author Yin Lou\n * \n */\npublic class ElasticNetLearner extends GLMLearner {\n\n\tstatic class Options extends LearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations (default: 0)\")\n\t\tint maxIter = 0;\n\n\t\t@Argument(name = \"-l\", description = \"lambda (default: 0)\")\n\t\tdouble lambda = 0;\n\n\t\t@Argument(name = \"-a\", description = \"L1 ratio (default: 0)\")\n\t\tdouble l1Ratio = 0;\n\n\t}\n\n\t/**\n\t * Trains elastic-net-regularized GLMs.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.glm.ElasticNetLearner\n\t * -t\ttrain set path\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-m]\tmaximum number of iterations (default: 0)\n\t * [-l]\tlambda (default: 0)\n\t * [-a]\tL1 ratio (default: 0)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(ElasticNetLearner.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tElasticNetLearner learner = new ElasticNetLearner();\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setTask(task);\n\t\tlearner.setLambda(opts.lambda);\n\t\tlearner.setL1Ratio(opts.l1Ratio);\n\t\tlearner.setMaxNumIters(opts.maxIter);\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGLM glm = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(glm, opts.outputModelPath);\n\t\t}\n\t}\n\n\tprotected double lambda;\n\tprotected double l1Ratio;\n\tprotected Task task;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic ElasticNetLearner() {\n\t\tlambda = 0; // no regularization\n\t\tl1Ratio = 0; // 0: ridge, 1: lasso, (0, 1): elastic net\n\t\ttask = Task.REGRESSION;\n\t}\n\n\t@Override\n\tpublic GLM build(Instances instances) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tglm = buildGaussianRegressor(instances, maxNumIters, lambda, l1Ratio);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tglm = buildClassifier(instances, maxNumIters, lambda, l1Ratio);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn glm;\n\t}\n\t\n\t@Override\n\tpublic GLM build(Instances trainSet, Family family) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (family) {\n\t\t\tcase GAUSSIAN:\n\t\t\t\tglm = buildGaussianRegressor(trainSet, maxNumIters, lambda, l1Ratio);\n\t\t\t\tbreak;\n\t\t\tcase BINOMIAL:\n\t\t\t\tglm = buildClassifier(trainSet, maxNumIters, lambda, l1Ratio);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tthrow new IllegalArgumentException(\"Unsupported family: \" + family);\n\t\t}\n\t\treturn glm;\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized binary classifier. Each row in the input matrix x represents a feature (instead\n\t * of a data point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized binary classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\tfinal double tl1 = lambda1 * y.length;\n\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t// Coordinate descent\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(x, theta, y, tl1, tl2, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized binary classifier on sparse inputs. Each row of the input represents a feature\n\t * (instead of a data point), i.e., in column-oriented format. This procedure does not assume the data is normalized\n\t * or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\tfinal double tl1 = lambda1 * y.length;\n\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t// Coordinate descent\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(indices, values, theta, y, tl1, tl2, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t}\n\n\t/**\n\t * Builds elastic-net penalized binary classifiers for a sequence of regularization parameter lambdas. Each row in\n\t * the input matrix x represents a feature (instead of a data point). Thus the input matrix is the transpose of the\n\t * row-oriented data matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return elastic-net penalized classifiers.\n\t */\n\tpublic GLM[] buildBinaryClassifiers(int[] attrs, double[][] x, double[] y, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(x, y, pTrain, rTrain, l1Ratio);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\tGLM[] glms = new GLM[numLambdas];\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < numLambdas; g++) {\n\t\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\t\tfinal double tl1 = lambda1 * y.length;\n\t\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(x, theta, y, tl1, tl2, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\tlambda *= alpha;\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds elastic-net penalized binary classifiers on sparse inputs for a sequence of regularization parameter\n\t * lambdas. Each row of the input represents a feature (instead of a data point), i.e., in column-oriented format.\n\t * This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized classifier.\n\t */\n\tpublic GLM[] buildBinaryClassifiers(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(indices, values, y, pTrain, rTrain, l1Ratio);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\tGLM[] glms = new GLM[numLambdas];\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < glms.length; g++) {\n\t\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\t\tfinal double tl1 = lambda1 * y.length;\n\t\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(indices, values, theta, y, tl1, tl2, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(pTrain, y, w, lambda1, lambda2);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\tlambda *= alpha;\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the l1 ratio.\n\t * @return an elastic-net penalized classifer.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, boolean isSparse, int maxNumIters, double lambda, double l1Ratio) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda, l1Ratio);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : attrs[attrs.length - 1] + 1;\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda,\n\t\t\t\t\t\t\tl1Ratio);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda, l1Ratio);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda, l1Ratio);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the l1 ratio.\n\t * @return an elastic-net penalized classifer.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, int maxNumIters, double lambda, double l1Ratio) {\n\t\treturn buildClassifier(trainSet, isSparse(trainSet), maxNumIters, lambda, l1Ratio);\n\t}\n\n\t/**\n\t * Builds elastic-net penalized classifiers.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the l1 ratio.\n\t * @return elastic-net penalized classifers.\n\t */\n\tpublic GLM[] buildClassifiers(Instances trainSet, boolean isSparse, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio, double l1Ratio) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM[] glms = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters, numLambdas, minLambdaRatio,\n\t\t\t\t\t\tl1Ratio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM[] glms = new GLM[numLambdas];\n\t\t\t\tfor (int i = 0; i < glms.length; i++) {\n\t\t\t\t\tglms[i] = new GLM(numClasses, p);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM[] binaryClassifiers = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters,\n\t\t\t\t\t\t\tnumLambdas, minLambdaRatio, l1Ratio);\n\n\t\t\t\t\tfor (int l = 0; l < glms.length; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers[l];\n\t\t\t\t\t\tGLM glm = glms[l];\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM[] glms = buildBinaryClassifiers(attrs, x, y, maxNumIters, numLambdas, minLambdaRatio, l1Ratio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM[] glms = new GLM[numLambdas];\n\t\t\t\tfor (int i = 0; i < glms.length; i++) {\n\t\t\t\t\tglms[i] = new GLM(numClasses, p);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM[] binaryClassifiers = buildBinaryClassifiers(attrs, x, y, maxNumIters, numLambdas,\n\t\t\t\t\t\t\tminLambdaRatio, l1Ratio);\n\n\t\t\t\t\tfor (int l = 0; l < glms.length; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers[l];\n\t\t\t\t\t\tGLM glm = glms[l];\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds elastic-net penalized classifiers.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the l1 ratio.\n\t * @return elastic-net penalized classifers.\n\t */\n\tpublic GLM[] buildClassifiers(Instances trainSet, int maxNumIters, int numLambdas, double minLambdaRatio,\n\t\t\tdouble l1Ratio) {\n\t\treturn buildClassifiers(trainSet, isSparse(trainSet), maxNumIters, numLambdas, minLambdaRatio, l1Ratio);\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, boolean isSparse, int maxNumIters, double lambda, double l1Ratio) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, lambda, l1Ratio);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\n\t\t\treturn glm;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(dd.attrs, dd.x, dd.y, maxNumIters, lambda, l1Ratio);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\t\t\treturn glm;\n\t\t}\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, int maxNumIters, double lambda, double l1Ratio) {\n\t\treturn buildGaussianRegressor(trainSet, isSparse(trainSet), maxNumIters, lambda, l1Ratio);\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized regressor. Each row in the input matrix x represents a feature (instead of a data\n\t * point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not assume\n\t * the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(x[i]);\n\t\t}\n\n\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\tfinal double tl1 = lambda1 * y.length;\n\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t// Coordinate descent\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(x, sq, tl1, tl2, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t}\n\n\t/**\n\t * Builds an elastic-net penalized regressor on sparse inputs. Each row of the input represents a feature (instead\n\t * of a data point), i.e., in column-oriented format. This procedure does not assume the data is normalized or\n\t * centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return an elastic-net penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[attrs.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(values[i]);\n\t\t}\n\n\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\tfinal double tl1 = lambda1 * y.length;\n\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t// Coordinate descent\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(indices, values, sq, tl1, tl2, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t}\n\n\t/**\n\t * Builds elastic-net penalized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return elastic-net penalized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(Instances trainSet, boolean isSparse, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio, double l1Ratio) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM[] glms = buildGaussianRegressors(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, numLambdas,\n\t\t\t\t\tminLambdaRatio, l1Ratio);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM[] glms = buildGaussianRegressors(dd.attrs, dd.x, dd.y, maxNumIters, numLambdas, minLambdaRatio, l1Ratio);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t}\n\t}\n\n\t/**\n\t * Builds elastic-net penalized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return elastic-net penalized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(Instances trainSet, int maxNumIters, int numLambdas, double minLambdaRatio,\n\t\t\tdouble l1Ratio) {\n\t\treturn buildGaussianRegressors(trainSet, isSparse(trainSet), maxNumIters, numLambdas, minLambdaRatio, l1Ratio);\n\t}\n\n\t/**\n\t * Builds elastic-net penalized regressors for a sequence of regularization parameter lambdas. Each row in the input\n\t * matrix x represents a feature (instead of a data point). Thus the input matrix is the transpose of the\n\t * row-oriented data matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return elastic-net penalized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(int[] attrs, double[][] x, double[] y, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tGLM[] glms = new GLM[numLambdas];\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(x[i]);\n\t\t}\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(x, y, l1Ratio);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < glms.length; g++) {\n\n\t\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\t\tfinal double tl1 = lambda1 * y.length;\n\t\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(x, sq, tl1, tl2, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds elastic-net penalized regressors on sparse inputs for a sequence of regularization parameter lambdas. Each row of the input\n\t * represents a feature (instead of a data point), i.e., in column-oriented format. This procedure does not assume\n\t * the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @param l1Ratio the L1 ratio.\n\t * @return elastic-net penalized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio, double l1Ratio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tGLM[] glms = new GLM[numLambdas];\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(values[i]);\n\t\t}\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(indices, values, y, l1Ratio);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < glms.length; g++) {\n\n\t\t\tfinal double lambda1 = lambda * l1Ratio;\n\t\t\tfinal double lambda2 = lambda * (1 - l1Ratio);\n\t\t\tfinal double tl1 = lambda1 * y.length;\n\t\t\tfinal double tl2 = lambda2 * y.length;\n\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(indices, values, sq, tl1, tl2, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeElasticNetLoss(rTrain, w, lambda1, lambda2);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\tlambda *= alpha;\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\tprotected void doOnePassGaussian(double[][] x, double[] sq, final double tl1, final double tl2, double[] w, double[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tdouble[] v = x[j];\n\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble wNew = w[j] * sq[j] + VectorUtils.dotProduct(v, rTrain);\n\t\t\tif (Math.abs(wNew) <= tl1) {\n\t\t\t\twNew = 0;\n\t\t\t} else if (wNew > 0) {\n\t\t\t\twNew -= tl1;\n\t\t\t} else {\n\t\t\t\twNew += tl1;\n\t\t\t}\n\t\t\twNew /= (sq[j] + tl2);\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\trTrain[i] -= delta * v[i];\n\t\t\t}\n\t\t}\n\t}\n\t\n\tprotected void doOnePassGaussian(int[][] indices, double[][] values, double[] sq, final double tl1, final double tl2,\n\t\t\tdouble[] w, double[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble wNew = w[j] * sq[j];\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\twNew += rTrain[index[i]] * value[i];\n\t\t\t}\n\n\t\t\tif (Math.abs(wNew) <= tl1) {\n\t\t\t\twNew = 0;\n\t\t\t} else if (wNew > 0) {\n\t\t\t\twNew -= tl1;\n\t\t\t} else {\n\t\t\t\twNew += tl1;\n\t\t\t}\n\t\t\twNew /= (sq[j] + tl2);\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\trTrain[index[i]] -= delta * value[i];\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected void doOnePassBinomial(double[][] x, double[] theta, double[] y, final double tl1, final double tl2, double[] w,\n\t\t\tdouble[] pTrain, double[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[] v = x[j];\n\t\t\tdouble eta = VectorUtils.dotProduct(rTrain, v);\n\n\t\t\tdouble newW = w[j] * theta[j] + eta;\n\t\t\tif (newW > tl1) {\n\t\t\t\tnewW -= tl1;\n\t\t\t} else if (newW < -tl1) {\n\t\t\t\tnewW += tl1;\n\t\t\t} else {\n\t\t\t\tnewW = 0;\n\t\t\t}\n\t\t\tnewW /= (theta[j] + tl2);\n\n\t\t\tdouble delta = newW - w[j];\n\t\t\tw[j] = newW;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < pTrain.length; i++) {\n\t\t\t\tpTrain[i] += delta * v[i];\n\t\t\t\trTrain[i] = OptimUtils.getPseudoResidual(pTrain[i], y[i]);\n\t\t\t}\n\t\t}\n\t}\n\t\n\tprotected void doOnePassBinomial(int[][] indices, double[][] values, double[] theta, double[] y, final double tl1,\n\t\t\tfinal double tl2, double[] w, double[] pTrain, double[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble eta = 0;\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\teta += rTrain[idx] * value[i];\n\t\t\t}\n\n\t\t\tdouble newW = w[j] * theta[j] + eta;\n\t\t\tif (newW > tl1) {\n\t\t\t\tnewW -= tl1;\n\t\t\t} else if (newW < -tl1) {\n\t\t\t\tnewW += tl1;\n\t\t\t} else {\n\t\t\t\tnewW = 0;\n\t\t\t}\n\t\t\tnewW /= (theta[j] + tl2);\n\n\t\t\tdouble delta = newW - w[j];\n\t\t\tw[j] = newW;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\tpTrain[idx] += delta * value[i];\n\t\t\t\trTrain[idx] = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected double findMaxLambdaGaussian(double[][] x, double[] y, double l1Ratio) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(y);\n\t\t}\n\t\t// Determine max lambda\n\t\tdouble maxLambda = 0;\n\t\tfor (double[] col : x) {\n\t\t\tdouble dot = Math.abs(VectorUtils.dotProduct(col, y));\n\t\t\tif (dot > maxLambda) {\n\t\t\t\tmaxLambda = dot;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tmaxLambda /= l1Ratio;\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(y, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\t\n\tprotected double findMaxLambdaGaussian(int[][] indices, double[][] values, double[] y, double l1Ratio) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(y);\n\t\t}\n\n\t\tDenseVector v = new DenseVector(y);\n\t\t// Determine max lambda\n\t\tdouble maxLambda = 0;\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tint[] index = indices[i];\n\t\t\tdouble[] value = values[i];\n\t\t\tdouble dot = Math.abs(VectorUtils.dotProduct(new SparseVector(index, value), v));\n\t\t\tif (dot > maxLambda) {\n\t\t\t\tmaxLambda = dot;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tmaxLambda /= l1Ratio;\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(y, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\tprotected double findMaxLambdaBinomial(double[][] x, double[] y, double[] pTrain, double[] rTrain, double l1Ratio) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (double[] col : x) {\n\t\t\tdouble eta = 0;\n\t\t\tfor (int i = 0; i < col.length; i++) {\n\t\t\t\teta += rTrain[i] * col[i];\n\t\t\t}\n\n\t\t\tdouble t = Math.abs(eta);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tmaxLambda /= l1Ratio;\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\tprotected double findMaxLambdaBinomial(int[][] indices, double[][] values, double[] y, double[] pTrain, double[] rTrain, double l1Ratio) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tdouble eta = 0;\n\t\t\tint[] index = indices[k];\n\t\t\tdouble[] value = values[k];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\tdouble r = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t\tr *= value[i];\n\t\t\t\teta += r;\n\t\t\t}\n\t\t\tdouble t = Math.abs(eta);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tmaxLambda /= l1Ratio;\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\t/**\n\t * Returns the l1 ratio.\n\t * \n\t * @return the l1 ratio.\n\t */\n\tpublic double getL1Ratio() {\n\t\treturn l1Ratio;\n\t}\n\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Sets the l1 ratio.\n\t * \n\t * @param l1Ratio the l1 ratio.\n\t */\n\tpublic void setL1Ratio(double l1Ratio) {\n\t\tthis.l1Ratio = l1Ratio;\n\t}\n\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/GLM.java",
    "content": "package mltk.predictor.glm;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.core.SparseVector;\nimport mltk.predictor.LinkFunction;\nimport mltk.predictor.ProbabilisticClassifier;\nimport mltk.predictor.Regressor;\nimport mltk.util.ArrayUtils;\nimport mltk.util.MathUtils;\nimport mltk.util.StatUtils;\n\n/**\n * Class for generalized linear models (GLMs).\n * \n * @author Yin Lou\n * \n */\npublic class GLM implements ProbabilisticClassifier, Regressor {\n\n\t/**\n\t * The coefficient vectors.\n\t */\n\tprotected double[][] w;\n\n\t/**\n\t * The intercept vector.\n\t */\n\tprotected double[] intercept;\n\t\n\t/**\n\t * The link function.\n\t */\n\tprotected LinkFunction link;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic GLM() {\n\n\t}\n\n\t/**\n\t * Constructs a GLM with the specified dimension.\n\t * \n\t * @param dimension the dimension.\n\t */\n\tpublic GLM(int dimension) {\n\t\tthis(1, dimension);\n\t}\n\n\t/**\n\t * Constructs a GLM with the specified dimension.\n\t * \n\t * @param numClasses the number of classes.\n\t * @param dimension the dimension.\n\t */\n\tpublic GLM(int numClasses, int dimension) {\n\t\tw = new double[numClasses][dimension];\n\t\tintercept = new double[numClasses];\n\t}\n\n\t/**\n\t * Constructs a GLM with the intercept vector and the coefficient vectors.\n\t * \n\t * @param intercept the intercept vector.\n\t * @param w the coefficient vectors.\n\t */\n\tpublic GLM(double[] intercept, double[][] w) {\n\t\tthis(intercept, w, LinkFunction.IDENTITY);\n\t}\n\t\n\t/**\n\t * Constructs a GLM with the intercept vector, the coefficient vectors and its link function.\n\t * \n\t * @param intercept the intercept vector.\n\t * @param w the coefficient vectors.\n\t * @param link the link function.\n\t */\n\tpublic GLM(double[] intercept, double[][] w, LinkFunction link) {\n\t\tif (intercept.length != w.length) {\n\t\t\tthrow new IllegalArgumentException(\"Dimensions of intercept and w must match.\");\n\t\t}\n\t\tthis.intercept = intercept;\n\t\tthis.w = w;\n\t\tthis.link = link;\n\t}\n\n\t/**\n\t * Returns the coefficient vectors.\n\t * \n\t * @return the coefficient vectors.\n\t */\n\tpublic double[][] coefficients() {\n\t\treturn w;\n\t}\n\n\t/**\n\t * Returns the coefficient vectors for class k.\n\t * \n\t * @param k the index of the class.\n\t * @return the coefficient vectors for class k.\n\t */\n\tpublic double[] coefficients(int k) {\n\t\treturn w[k];\n\t}\n\n\t/**\n\t * Returns the intercept vector.\n\t * \n\t * @return the intercept vector.\n\t */\n\tpublic double[] intercept() {\n\t\treturn intercept;\n\t}\n\n\t/**\n\t * Returns the intercept for class k.\n\t * \n\t * @param k the index of the class.\n\t * @return the intercept for class k.\n\t */\n\tpublic double intercept(int k) {\n\t\treturn intercept[k];\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tlink = LinkFunction.get(in.readLine().split(\": \")[1]);\n\t\tin.readLine();\n\t\tintercept = ArrayUtils.parseDoubleArray(in.readLine());\n\t\tint p = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tw = new double[intercept.length][p];\n\t\tfor (int j = 0; j < p; j++) {\n\t\t\tString[] data = in.readLine().split(\"\\\\s+\");\n\t\t\tfor (int i = 0; i < w.length; i++) {\n\t\t\t\tw[i][j] = Double.parseDouble(data[i]);\n\t\t\t}\n\t\t}\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Link: \" + link);\n\t\tout.println(\"Intercept: \" + intercept.length);\n\t\tout.println(Arrays.toString(intercept));\n\t\tfinal int p = w[0].length;\n\t\tout.println(\"Coefficients: \" + p);\n\t\tfor (int j = 0; j < p; j++) {\n\t\t\tout.print(w[0][j]);\n\t\t\tfor (int i = 1; i < w.length; i++) {\n\t\t\t\tout.print(\" \" + w[i][j]);\n\t\t\t}\n\t\t\tout.println();\n\t\t}\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\treturn regress(intercept[0], w[0], instance);\n\t}\n\n\t@Override\n\tpublic int classify(Instance instance) {\n\t\tdouble[] prob = predictProbabilities(instance);\n\t\treturn StatUtils.indexOfMax(prob);\n\t}\n\t\n\t/**\n\t * Returns the prediction of this GLM on the scale of the response variable.\n\t * \n\t * @param instance the instance to predict.\n\t * @return the prediction of this GLM on the scale of the response variable.\n\t */\n\tpublic double predict(Instance instance) {\n\t\treturn link.applyInverse(regress(instance));\n\t}\n\n\t@Override\n\tpublic double[] predictProbabilities(Instance instance) {\n\t\tif (w.length == 1) {\n\t\t\tdouble[] prob = new double[2];\n\t\t\tdouble pred = regress(intercept[0], w[0], instance);\n\t\t\tprob[0] = MathUtils.sigmoid(pred);\n\t\t\tprob[1] = 1 - prob[0];\n\t\t\treturn prob;\n\t\t} else {\n\t\t\tdouble[] prob = new double[w.length];\n\t\t\tdouble[] pred = new double[w.length];\n\t\t\tdouble sum = 0;\n\t\t\tfor (int i = 0; i < w.length; i++) {\n\t\t\t\tpred[i] = regress(intercept[i], w[i], instance);\n\t\t\t\tprob[i] = MathUtils.sigmoid(pred[i]);\n\t\t\t\tsum += prob[i];\n\t\t\t}\n\t\t\tfor (int i = 0; i < prob.length; i++) {\n\t\t\t\tprob[i] /= sum;\n\t\t\t}\n\t\t\treturn prob;\n\t\t}\n\t}\n\n\t@Override\n\tpublic GLM copy() {\n\t\tdouble[][] copyW = new double[w.length][];\n\t\tfor (int i = 0; i < copyW.length; i++) {\n\t\t\tcopyW[i] = Arrays.copyOf(w[i], w[i].length);\n\t\t}\n\t\treturn new GLM(intercept, copyW, link);\n\t}\n\n\tprotected double regress(double intercept, double[] coef, Instance instance) {\n\t\tif (!instance.isSparse()) {\n\t\t\tdouble pred = intercept;\n\t\t\tfor (int i = 0; i < coef.length; i++) {\n\t\t\t\tpred += coef[i] * instance.getValue(i);\n\t\t\t}\n\t\t\treturn pred;\n\t\t} else {\n\t\t\tdouble pred = intercept;\n\t\t\tSparseVector vector = (SparseVector) instance.getVector();\n\t\t\tint[] indices = vector.getIndices();\n\t\t\tdouble[] values = vector.getValues();\n\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\tint index = indices[i];\n\t\t\t\tif (index < coef.length) {\n\t\t\t\t\tpred += coef[index] * values[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn pred;\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/GLMLearner.java",
    "content": "package mltk.predictor.glm;\n\nimport mltk.core.Instances;\nimport mltk.predictor.Family;\nimport mltk.predictor.Learner;\nimport mltk.util.MathUtils;\n\n/**\n * Abstract class for learning generalized linear models (GLMs).\n * \n * @author Yin Lou\n *\n */\npublic abstract class GLMLearner extends Learner {\n\t\n\tprotected boolean fitIntercept;\n\tprotected int maxNumIters;\n\tprotected double epsilon;\n\tprotected Family family;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic GLMLearner() {\n\t\tverbose = false;\n\t\tfitIntercept = true;\n\t\tmaxNumIters = -1;\n\t\tepsilon = MathUtils.EPSILON;\n\t\tfamily = Family.GAUSSIAN;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if we fit intercept.\n\t * \n\t * @return {@code true} if we fit intercept.\n\t */\n\tpublic boolean fitIntercept() {\n\t\treturn fitIntercept;\n\t}\n\n\t/**\n\t * Sets whether we fit intercept.\n\t * \n\t * @param fitIntercept whether we fit intercept.\n\t */\n\tpublic void fitIntercept(boolean fitIntercept) {\n\t\tthis.fitIntercept = fitIntercept;\n\t}\n\n\t/**\n\t * Returns the convergence threshold epsilon.\n\t * \n\t * @return the convergence threshold epsilon.\n\t */\n\tpublic double getEpsilon() {\n\t\treturn epsilon;\n\t}\n\t\n\t/**\n\t * Sets the convergence threshold epsilon.\n\t * \n\t * @param epsilon the convergence threshold epsilon.\n\t */\n\tpublic void setEpsilon(double epsilon) {\n\t\tthis.epsilon = epsilon;\n\t}\n\t\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\t\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\t\n\t/**\n\t * Returns the response distribution family.\n\t * \n\t * @return the response distribution family.\n\t */\n\tpublic Family getFamily() {\n\t\treturn family;\n\t}\n\t\n\t/**\n\t * Sets the response distribution family.\n\t * \n\t * @param family the response distribution family.\n\t */\n\tpublic void setFamily(Family family) {\n\t\tthis.family = family;\n\t}\n\t\n\t/**\n\t * Builds a generalized linear model given response distribution family.\n\t * The default link function for the family will be used.\n\t * \n\t * @param trainSet the training set.\n\t * @param family the response distribution family.\n\t * @return a generalized linear model.\n\t */\n\tpublic abstract GLM build(Instances trainSet, Family family);\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/GLMOptimUtils.java",
    "content": "package mltk.predictor.glm;\r\n\r\nimport mltk.predictor.LinkFunction;\r\nimport mltk.util.OptimUtils;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.VectorUtils;\r\n\r\nclass GLMOptimUtils {\r\n\r\n\tstatic GLM getGLM(int[] attrs, double[] w, double intercept, LinkFunction link) {\r\n\t\tfinal int p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\r\n\t\tGLM glm = new GLM(p);\r\n\t\tfor (int i = 0; i < attrs.length; i++) {\r\n\t\t\tglm.w[0][attrs[i]] = w[i];\r\n\t\t}\r\n\t\tglm.intercept[0] = intercept;\r\n\t\tglm.link = link;\r\n\t\treturn glm;\r\n\t}\r\n\r\n\tstatic double computeRidgeLoss(double[] residual, double[] w, double lambda) {\r\n\t\tdouble loss =  OptimUtils.computeQuadraticLoss(residual);\r\n\t\tloss += lambda / 2 * StatUtils.sumSq(w);\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeRidgeLoss(double[] pred, double[] y, double[] w, double lambda) {\r\n\t\tdouble loss = OptimUtils.computeLogisticLoss(pred, y);\r\n\t\tloss += lambda / 2 * StatUtils.sumSq(w);\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeLassoLoss(double[] residual, double[] w, double lambda) {\r\n\t\tdouble loss =  OptimUtils.computeQuadraticLoss(residual);\r\n\t\tloss += lambda * VectorUtils.l1norm(w);\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeLassoLoss(double[] pred, double[] y, double[] w, double lambda) {\r\n\t\tdouble loss =  OptimUtils.computeLogisticLoss(pred, y);\r\n\t\tloss += lambda * VectorUtils.l1norm(w);\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeElasticNetLoss(double[] residual, double[] w, double lambda1, double lambda2) {\r\n\t\tdouble loss =  OptimUtils.computeQuadraticLoss(residual);\r\n\t\tloss += lambda1 * VectorUtils.l1norm(w) + lambda2 / 2 * StatUtils.sumSq(w);\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeElasticNetLoss(double[] pred, double[] y, double[] w, double lambda1, double lambda2) {\r\n\t\tdouble loss =  OptimUtils.computeLogisticLoss(pred, y);\r\n\t\tloss += lambda1 * VectorUtils.l1norm(w) + lambda2 / 2 * StatUtils.sumSq(w);\r\n\t\treturn loss;\r\n\t}\r\n\t\r\n\tstatic double computeGroupLassoLoss(double[] residual, double[][] w, double[] tl1) {\r\n\t\tdouble loss =  OptimUtils.computeQuadraticLoss(residual);\r\n\t\tfor (int k = 0; k < w.length; k++) {\r\n\t\t\tloss += tl1[k] * StatUtils.sumSq(w[k]);\r\n\t\t}\r\n\t\treturn loss;\r\n\t}\r\n\r\n\tstatic double computeGroupLassoLoss(double[] pred, double[] y, double[][] w, double[] tl1) {\r\n\t\tdouble loss =  OptimUtils.computeLogisticLoss(pred, y);\r\n\t\tfor (int k = 0; k < w.length; k++) {\r\n\t\t\tloss += tl1[k] * StatUtils.sumSq(w[k]);\r\n\t\t}\r\n\t\treturn loss;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/GroupLassoLearner.java",
    "content": "package mltk.predictor.glm;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.predictor.Family;\nimport mltk.predictor.LinkFunction;\nimport mltk.predictor.glm.GLM;\nimport mltk.util.ArrayUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for learning group-lasso models via block coordinate gradient descent.\n * \n * <p>\n * Reference:<br>\n * M Yuan and Y Lin. Model selection and estimation in regression with grouped variables. In \n * <i>Journal of the Royal Statistical Society: Series B (Statistical Methodology)</i>, \n * 68(1):49-67, 2006.\n * </p>\n * \n * @author Yin Lou\n * \n */\npublic class GroupLassoLearner extends GLMLearner {\n\t\n\tclass DenseDesignMatrix {\n\n\t\tint[][] groups;\n\t\tdouble[][][] x;\n\n\t\tDenseDesignMatrix(int[][] groups, double[][][] x) {\n\t\t\tthis.groups = groups;\n\t\t\tthis.x = x;\n\t\t}\n\n\t}\n\t\n\tstatic class ModelStructure {\n\n\t\tboolean[] structure;\n\n\t\tModelStructure(boolean[] structure) {\n\t\t\tthis.structure = structure;\n\t\t}\n\n\t\t@Override\n\t\tpublic boolean equals(Object obj) {\n\t\t\tif (this == obj)\n\t\t\t\treturn true;\n\t\t\tif (obj == null)\n\t\t\t\treturn false;\n\t\t\tif (getClass() != obj.getClass())\n\t\t\t\treturn false;\n\t\t\tModelStructure other = (ModelStructure) obj;\n\t\t\tif (!Arrays.equals(structure, other.structure))\n\t\t\t\treturn false;\n\t\t\treturn true;\n\t\t}\n\n\t\t@Override\n\t\tpublic int hashCode() {\n\t\t\tfinal int prime = 31;\n\t\t\tint result = 1;\n\t\t\tresult = prime * result + Arrays.hashCode(structure);\n\t\t\treturn result;\n\t\t}\n\n\t}\n\t\n\tclass SparseDesignMatrix {\n\n\t\tint[][] group;\n\t\tint[][][] indices;\n\t\tdouble[][][] values;\n\n\t\tSparseDesignMatrix(int[][] groups, int[][][] indices, double[][][] values) {\n\t\t\tthis.indices = indices;\n\t\t\tthis.values = values;\n\t\t}\n\t}\n\n\tprotected boolean refit;\n\tprotected int numLambdas;\n\tprotected double lambda;\n\tprotected Task task;\n\tprotected List<int[]> groups;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic GroupLassoLearner() {\n\t\trefit = false;\n\t\tlambda = 0.0;\n\t\ttask = Task.REGRESSION;\n\t\tgroups = null;\n\t}\n\n\t@Override\n\tpublic GLM build(Instances instances) {\n\t\tif (groups == null) {\n\t\t\tthrow new IllegalArgumentException(\"Groups are not set.\");\n\t\t}\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tglm = buildGaussianRegressor(instances, groups, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tglm = buildClassifier(instances, groups, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn glm;\n\t}\n\t\n\t@Override\n\tpublic GLM build(Instances trainSet, Family family) {\n\t\tif (groups == null) {\n\t\t\tthrow new IllegalArgumentException(\"Groups are not set.\");\n\t\t}\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (family) {\n\t\t\tcase GAUSSIAN:\n\t\t\t\tglm = buildGaussianRegressor(trainSet, groups, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase BINOMIAL:\n\t\t\t\tglm = buildClassifier(trainSet, groups, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tthrow new IllegalArgumentException(\"Unsupported family: \" + family);\n\t\t}\n\t\treturn glm;\n\t}\n\n\t/**\n\t * Builds a group-lasso penalized binary classifier. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the groups of variables.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[][] attrs, double[][][] x, double[] y, int maxNumIters, double lambda) {\n\t\tint p = 0;\n\t\tif (attrs.length > 0) {\n\t\t\tfor (int[] attr : attrs) {\n\t\t\t\tp = Math.max(p, StatUtils.max(attr));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[attrs[j].length];\n\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tboolean[] activeSet = new boolean[attrs.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePassBinomial(x, y, tl1, true, activeSet, w, stepSize, \n\t\t\t\t\tg, gradient, pTrain, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(x, y, tl1, false, activeSet, w, stepSize, g, gradient, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t}\n\t\t\treturn refitGaussianRegressor(p, attrs, selected, x, y, w, maxNumIters);\n\t\t} else {\n\t\t\treturn getGLM(p, attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\t}\n\n\t/**\n\t * Builds a group-lasso penalized binary classifier on sparse inputs. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the groups of variables.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[][] attrs, int[][][] indices, double[][][] values, double[] y, int maxNumIters, double lambda) {\n\t\tint p = 0;\n\t\tif (attrs.length > 0) {\n\t\t\tfor (int[] attr : attrs) {\n\t\t\t\tp = Math.max(p, StatUtils.max(attr));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\tint[][] indexUnion = new int[attrs.length][];\n\t\tfor (int g = 0; g < attrs.length; g++) {\n\t\t\tint[][] index = indices[g];\n\t\t\tSet<Integer> set = new HashSet<>();\n\t\t\tfor (int[] idx : index) {\n\t\t\t\tfor (int i : idx) {\n\t\t\t\t\tset.add(i);\n\t\t\t\t}\n\t\t\t}\n\t\t\tint[] idxUnion = new int[set.size()];\n\t\t\tint k = 0;\n\t\t\tfor (int idx : set) {\n\t\t\t\tidxUnion[k++] = idx;\n\t\t\t}\n\t\t\tindexUnion[g] = idxUnion;\n\t\t}\n\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[attrs[j].length];\n\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tboolean[] activeSet = new boolean[values.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePassBinomial(indices, indexUnion, values, y, tl1, true, \n\t\t\t\t\tactiveSet, w, stepSize, g, gradient, pTrain, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(indices, indexUnion, values, y, tl1, true, activeSet, w, stepSize, \n\t\t\t\t\t\tg, gradient, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t}\n\t\t\treturn refitClassifier(p, attrs, selected, indices, values, y, w, maxNumIters);\n\t\t} else {\n\t\t\treturn getGLM(p, attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\t}\n\n\t/**\n\t * Builds group-lasso penalized binary classifiers for a sequence of regularization parameter lambdas. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the groups of variables.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized classifiers.\n\t */\n\tpublic List<GLM> buildBinaryClassifiers(int[][] attrs, double[][][] x, double[] y, int maxNumIters, int numLambdas, double minLambdaRatio) {\n\t\tint p = 0;\n\t\tif (attrs.length > 0) {\n\t\t\tfor (int[] attr : attrs) {\n\t\t\t\tp = Math.max(p, StatUtils.max(attr));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[attrs[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tdouble[] tl1 = new double[x.length];\n\n\t\tdouble[] stepSize = new double[x.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\n\t\tboolean[] activeSet = new boolean[x.length];\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(x, y, pTrain, rTrain, gradient);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tdouble lambda = maxLambda;\n\t\tdouble intercept = 0;\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t// Initialize regularization parameters\n\t\t\tfor (int j = 0; j < tl1.length; j++) {\n\t\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\t}\n\n\t\t\t// Block coordinate gradient descent\n\t\t\tint iter = 0;\n\t\t\twhile (iter < maxNumIters) {\n\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tboolean activeSetChanged = doOnePassBinomial(x, y, tl1, true, activeSet, w,\n\t\t\t\t\t\tstepSize, g, gradient, pTrain, rTrain);\n\n\t\t\t\titer++;\n\n\t\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\t\n\t\t\t\t\tif (fitIntercept) {\n\t\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tdoOnePassBinomial(x, y, tl1, false, activeSet, w, stepSize, g, gradient, pTrain, rTrain);\n\n\t\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\n\t\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (verbose) {\n\t\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitClassifier(p, attrs, selected, x, y, w, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = getGLM(p, attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds group-lasso penalized binary classifiers on sparse inputs for a sequence of regularization parameter lambdas. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the groups of variables.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized classifiers.\n\t */\n\tpublic List<GLM> buildBinaryClassifiers(int[][] attrs, int[][][] indices, double[][][] values, double[] y, \n\t\t\tint maxNumIters, int numLambdas, double minLambdaRatio) {\n\t\tint p = 0;\n\t\tif (attrs.length > 0) {\n\t\t\tfor (int[] attr : attrs) {\n\t\t\t\tp = Math.max(p, StatUtils.max(attr));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\tint[][] indexUnion = new int[indices.length][];\n\t\tfor (int g = 0; g < indices.length; g++) {\n\t\t\tint[][] index = indices[g];\n\t\t\tSet<Integer> set = new HashSet<>();\n\t\t\tfor (int[] idx : index) {\n\t\t\t\tfor (int i : idx) {\n\t\t\t\t\tset.add(i);\n\t\t\t\t}\n\t\t\t}\n\t\t\tint[] idxUnion = new int[set.size()];\n\t\t\tint k = 0;\n\t\t\tfor (int idx : set) {\n\t\t\t\tidxUnion[k++] = idx;\n\t\t\t}\n\t\t\tindexUnion[g] = idxUnion;\n\t\t}\n\t\t\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < values.length; j++) {\n\t\t\tw[j] = new double[values[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t\n\t\tdouble[] stepSize = new double[values.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t) / 4;\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tboolean[] activeSet = new boolean[values.length];\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(indices, values, y, pTrain, rTrain, gradient);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\n\t\tdouble lambda = maxLambda;\n\t\tdouble intercept = 0;\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t// Initialize regularization parameters\n\t\t\tfor (int j = 0; j < tl1.length; j++) {\n\t\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\t}\n\n\t\t\t// Block coordinate gradient descent\n\t\t\tint iter = 0;\n\t\t\twhile (iter < maxNumIters) {\n\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tboolean activeSetChanged = doOnePassBinomial(indices, indexUnion, values, y, tl1, true, \n\t\t\t\t\t\tactiveSet, w, stepSize, g, gradient, pTrain, rTrain);\n\n\t\t\t\titer++;\n\n\t\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\t\n\t\t\t\t\tif (fitIntercept) {\n\t\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t\t}\n\n\t\t\t\t\tdoOnePassBinomial(indices, indexUnion, values, y, tl1, false, activeSet, w, stepSize,\n\t\t\t\t\t\t\tg, gradient, pTrain, rTrain);\n\n\t\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(pTrain, y, w, tl1);\n\t\t\t\t\t\n\t\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (verbose) {\n\t\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitClassifier(p, attrs, selected, indices, values, y, w, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = getGLM(p, attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\t\n\t/**\n\t * Builds a group-lasso penalized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized classifier.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, boolean isSparse, List<int[]> groups, int maxNumIters, double lambda) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\t\t\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tSparseDesignMatrix sm = createDesignMatrix(sd, groups);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] group = sm.group;\n\t\t\tint[][][] indices = sm.indices;\n\t\t\tdouble[][][] values = sm.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\t\t\t\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tGLM glm = buildBinaryClassifier(group, indices, values, y, maxNumIters, lambda);\n\t\t\t\t\n\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(group, indices, values, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tDenseDesignMatrix dm = createDesignMatrix(dd, groups);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tint[][] group = dm.groups;\n\t\t\tdouble[][][] x = dm.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\t\t\t\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tGLM glm = buildBinaryClassifier(group, x, y, maxNumIters, lambda);\n\n\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(group, x, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds a group-lasso penalized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized classifier.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, List<int[]> groups, int maxNumIters, double lambda) {\n\t\treturn buildClassifier(trainSet, isSparse(trainSet), groups, maxNumIters, lambda);\n\t}\n\t\n\t/**\n\t * Builds group-lasso penalized classifiers.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized classifiers.\n\t */\n\tpublic List<GLM> buildClassifiers(Instances trainSet, boolean isSparse, List<int[]> groups, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\t\t\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tSparseDesignMatrix sm = createDesignMatrix(sd, groups);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] group = sm.group;\n\t\t\tint[][][] indices = sm.indices;\n\t\t\tdouble[][][] values = sm.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tList<GLM> glms = buildBinaryClassifiers(sm.group, sm.indices, sm.values, y, maxNumIters, numLambdas,\n\t\t\t\t\t\tminLambdaRatio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tboolean refit = this.refit;\n\t\t\t\tthis.refit = false; // Not supported in multiclass\n\t\t\t\t\t\t\t\t\t// classification\n\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tList<GLM> glms = new ArrayList<>();\n\t\t\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t\t\tGLM glm = new GLM(numClasses, p);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tList<GLM> binaryClassifiers = buildBinaryClassifiers(group, indices, values, y, maxNumIters,\n\t\t\t\t\t\t\tnumLambdas, minLambdaRatio);\n\n\t\t\t\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers.get(l);\n\t\t\t\t\t\tGLM glm = glms.get(l);\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tthis.refit = refit;\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tDenseDesignMatrix dm = createDesignMatrix(dd, groups);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tint[][] group = dm.groups;\n\t\t\tdouble[][][] x = dm.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\t\t\t\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tList<GLM> glms = buildBinaryClassifiers(dm.groups, dm.x, y, maxNumIters, numLambdas, minLambdaRatio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tboolean refit = this.refit;\n\t\t\t\tthis.refit = false; // Not supported in multiclass\n\t\t\t\t\t\t\t\t\t// classification\n\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tList<GLM> glms = new ArrayList<>();\n\t\t\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t\t\tGLM glm = new GLM(numClasses, p);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tList<GLM> binaryClassifiers = buildBinaryClassifiers(group, x, y, maxNumIters, numLambdas,\n\t\t\t\t\t\t\tminLambdaRatio);\n\n\t\t\t\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers.get(l);\n\t\t\t\t\t\tGLM glm = glms.get(l);\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tthis.refit = refit;\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds group-lasso penalized classifiers.\n\t * \n\t * @param trainSet the training set.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized classifiers.\n\t */\n\tpublic List<GLM> buildClassifiers(Instances trainSet, List<int[]> groups, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\treturn buildClassifiers(trainSet, isSparse(trainSet), groups, maxNumIters, numLambdas, minLambdaRatio);\n\t}\n\t\n\t/**\n\t * Builds a group-lasso penalized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, boolean isSparse, List<int[]> groups, int maxNumIters, double lambda) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tSparseDesignMatrix sm = createDesignMatrix(sd, groups);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(sm.group, sm.indices, sm.values, sd.y, maxNumIters, lambda);\n\t\t\t\n\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\n\t\t\treturn glm;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tDenseDesignMatrix dm = createDesignMatrix(dd, groups);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(dm.groups, dm.x, dd.y, maxNumIters, lambda);\n\n\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\n\t\t\treturn glm;\n\t\t}\n\t}\n\n\t/**\n\t * Builds a group-lasso penalized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param groups the groups.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, List<int[]> groups, int maxNumIters, double lambda) {\n\t\treturn buildGaussianRegressor(trainSet, isSparse(trainSet), groups, maxNumIters, lambda);\n\t}\n\n\t/**\n\t * Builds a group-lasso penalized regressor. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the groups of variables.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[][] attrs, double[][][] x, double[] y, int maxNumIters, double lambda) {\n\t\tint p = 0;\n\t\tif (attrs.length > 0) {\n\t\t\tfor (int[] attr : attrs) {\n\t\t\t\tp = Math.max(p, StatUtils.max(attr));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\tdouble[][] w = new double[attrs.length][];\n\t\tdouble[] tl1 = new double[attrs.length];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tw[j] = new double[x[j].length];\n\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] stepSize = new double[attrs.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t);\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tboolean[] activeSet = new boolean[x.length];\n\n\t\tdouble intercept = 0;\n\t\t\n\t\t// Block coordinate gradient descent\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePassGaussian(x, tl1, true, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tdoOnePassGaussian(x, tl1, false, activeSet, w, stepSize, g, gradient, rTrain);\n\t\t\t\t\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t}\n\t\t\treturn refitGaussianRegressor(p, attrs, selected, x, y, w, maxNumIters);\n\t\t} else {\n\t\t\treturn getGLM(p, attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\t}\n\t\n\t/**\n\t * Builds a group-lasso penalized regressor on sparse inputs. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param groups the groups of variables.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return a group-lasso penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[][] groups, int[][][] indices, double[][][] values, double[] y, int maxNumIters, double lambda) {\n\t\tint p = 0;\n\t\tif (groups.length > 0) {\n\t\t\tfor (int[] group : groups) {\n\t\t\t\tp = Math.max(p, StatUtils.max(group));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\tdouble[][] w = new double[groups.length][];\n\t\tdouble[] tl1 = new double[groups.length];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < groups.length; j++) {\n\t\t\tw[j] = new double[groups[j].length];\n\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] stepSize = new double[groups.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t);\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tboolean[] activeSet = new boolean[groups.length];\n\n\t\tdouble intercept = 0;\n\t\tint iter = 0;\n\t\twhile (iter < maxNumIters) {\n\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tboolean activeSetChanged = doOnePassGaussian(indices, values, tl1, true, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\titer++;\n\n\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tdoOnePassGaussian(indices, values, tl1, false, activeSet, w, stepSize, g, gradient, rTrain);\n\t\t\t\t\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[groups.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = !ArrayUtils.isConstant(w[i], 0, w[i].length, 0);\n\t\t\t}\n\t\t\treturn refitGaussianRegressor(p, groups, selected, indices, values, y, w, maxNumIters);\n\t\t} else {\n\t\t\treturn getGLM(p, groups, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\t}\n\t\n\t/**\n\t * Builds group-lasso penalized regressors.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(Instances trainSet, boolean isSparse, List<int[]> groups, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tSparseDesignMatrix sm = createDesignMatrix(sd, groups);\n\t\t\tdouble[] cList = sd.cList;\n\t\t\t\n\t\t\tList<GLM> glms = buildGaussianRegressors(sm.group, sm.indices, sm.values, sd.y, maxNumIters, numLambdas, minLambdaRatio);\n\t\t\t\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\treturn glms;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, false);\n\t\t\tDenseDesignMatrix dm = createDesignMatrix(dd, groups);\n\t\t\tdouble[] stdList = dd.stdList;\n\n\t\t\tList<GLM> glms = buildGaussianRegressors(dm.groups, dm.x, dd.y, maxNumIters, numLambdas, minLambdaRatio);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.coefficients(0);\n\t\t\t\tfor (int j = 0; j < stdList.length; j++) {\n\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\tw[attIndex] *= stdList[j];\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\treturn glms;\n\t\t}\n\t}\n\n\t/**\n\t * Builds group-lasso penalized regressors.\n\t * \n\t * @param trainSet the training set.\n\t * @param groups the groups of variables.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(Instances trainSet, List<int[]> groups, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\treturn buildGaussianRegressors(trainSet, isSparse(trainSet), groups, maxNumIters, numLambdas, minLambdaRatio);\n\t}\n\n\t/**\n\t * Builds group-lasso penalized regressors for a sequence of regularization parameter lambdas. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param groups the groups of variables.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(int[][] groups, double[][][] x, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tint p = 0;\n\t\tif (groups.length > 0) {\n\t\t\tfor (int[] group : groups) {\n\t\t\t\tp = Math.max(p, StatUtils.max(group));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Allocate coefficients\n\t\tdouble[][] w = new double[x.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tw[j] = new double[x[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tdouble[] tl1 = new double[x.length];\n\n\t\tdouble[] stepSize = new double[x.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = x[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t);\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\n\t\tboolean[] activeSet = new boolean[x.length];\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(x, rTrain, gradient);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tdouble lambda = maxLambda;\n\t\tdouble intercept = 0;\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t// Initialize regularization parameters\n\t\t\tfor (int j = 0; j < tl1.length; j++) {\n\t\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\t}\n\n\t\t\t// Block coordinate gradient descent\n\t\t\tint iter = 0;\n\t\t\twhile (iter < maxNumIters) {\n\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tboolean activeSetChanged = doOnePassGaussian(x, tl1, true, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\t\titer++;\n\n\t\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\t\t\t\t\t\n\t\t\t\t\tif (fitIntercept) {\n\t\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tdoOnePassGaussian(x, tl1, false, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\n\t\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (verbose) {\n\t\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean allActivated = true;\n\t\t\t\tboolean[] selected = new boolean[groups.length];\n\t\t\t\tfor (int j = 0; j < selected.length; j++) {\n\t\t\t\t\tselected[j] = !ArrayUtils.isConstant(w[j], 0, w[j].length, 0);\n\t\t\t\t\tallActivated = allActivated & selected[j];\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitGaussianRegressor(p, groups, selected, x, y, w, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t\tif (allActivated) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = getGLM(p, groups, w, intercept, LinkFunction.IDENTITY);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\t\n\t/**\n\t * Builds group-lasso penalized regressors on sparse inputs for a sequence of regularization parameter lambdas. The input matrix is grouped by groups. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param groups the groups of variables.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return group-lasso penalized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(int[][] groups, int[][][] indices, double[][][] values, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tint p = 0;\n\t\tif (groups.length > 0) {\n\t\t\tfor (int[] group : groups) {\n\t\t\t\tp = Math.max(p, StatUtils.max(group));\n\t\t\t}\n\t\t\tp += 1;\n\t\t}\n\t\t\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Allocate coefficients\n\t\tdouble[][] w = new double[groups.length][];\n\t\tint m = 0;\n\t\tfor (int j = 0; j < groups.length; j++) {\n\t\t\tw[j] = new double[groups[j].length];\n\t\t\tif (w[j].length > m) {\n\t\t\t\tm = w[j].length;\n\t\t\t}\n\t\t}\n\n\t\tdouble[] g = new double[m];\n\t\tdouble[] gradient = new double[m];\n\n\t\tdouble[] tl1 = new double[groups.length];\n\n\t\tdouble[] stepSize = new double[groups.length];\n\t\tfor (int j = 0; j < stepSize.length; j++) {\n\t\t\tdouble max = 0;\n\t\t\tdouble[][] block = values[j];\n\t\t\tfor (double[] t : block) {\n\t\t\t\tdouble l = StatUtils.sumSq(t);\n\t\t\t\tif (l > max) {\n\t\t\t\t\tmax = l;\n\t\t\t\t}\n\t\t\t}\n\t\t\tstepSize[j] = 1.0 / max;\n\t\t}\n\n\t\tboolean[] activeSet = new boolean[groups.length];\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(indices, values, rTrain, gradient);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tdouble lambda = maxLambda;\n\t\tdouble intercept = 0;\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t// Initialize regularization parameters\n\t\t\tfor (int j = 0; j < tl1.length; j++) {\n\t\t\t\ttl1[j] = lambda * Math.sqrt(w[j].length);\n\t\t\t}\n\n\t\t\t// Block coordinate gradient descent\n\t\t\tint iter = 0;\n\t\t\twhile (iter < maxNumIters) {\n\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tboolean activeSetChanged = doOnePassGaussian(indices, values, tl1, true, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\t\titer++;\n\n\t\t\t\tif (!activeSetChanged || iter > maxNumIters) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tfor (; iter < maxNumIters; iter++) {\n\n\t\t\t\t\tdouble prevLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\t\t\t\t\t\n\t\t\t\t\tif (fitIntercept) {\n\t\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tdoOnePassGaussian(indices, values, tl1, false, activeSet, w, stepSize, g, gradient, rTrain);\n\n\t\t\t\t\tdouble currLoss = GLMOptimUtils.computeGroupLassoLoss(rTrain, w, tl1);\n\n\t\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (verbose) {\n\t\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + currLoss);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean allActivated = true;\n\t\t\t\tboolean[] selected = new boolean[groups.length];\n\t\t\t\tfor (int j = 0; j < selected.length; j++) {\n\t\t\t\t\tselected[j] = !ArrayUtils.isConstant(w[j], 0, w[j].length, 0);\n\t\t\t\t\tallActivated = allActivated & selected[j];\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitGaussianRegressor(p, groups, selected, indices, values, y, w, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t\tif (allActivated) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = getGLM(p, groups, w, intercept, LinkFunction.IDENTITY);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\n\t/**\n\t * Returns the number of lambdas.\n\t * \n\t * @return the number of lambdas.\n\t */\n\tpublic int getNumLambdas() {\n\t\treturn numLambdas;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Returns {@code true} if we refit the model.\n\t * \n\t * @return {@code true} if we refit the model.\n\t */\n\tpublic boolean refit() {\n\t\treturn refit;\n\t}\n\n\t/**\n\t * Sets whether we refit the model.\n\t * \n\t * @param refit {@code true} if we refit the model.\n\t */\n\tpublic void refit(boolean refit) {\n\t\tthis.refit = refit;\n\t}\n\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\n\t/**\n\t * Sets the number of lambdas.\n\t * \n\t * @param numLambdas the number of lambdas.\n\t */\n\tpublic void setNumLambdas(int numLambdas) {\n\t\tthis.numLambdas = numLambdas;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\t\n\t/**\n\t * Returns the groups.\n\t * \n\t * @return the groups.\n\t */\n\tpublic List<int[]> getGroups() {\n\t\treturn groups;\n\t}\n\t\n\t/**\n\t * Sets the groups.\n\t * \n\t * @param groups the groups.\n\t */\n\tpublic void setGroups(List<int[]> groups) {\n\t\tthis.groups = groups;\n\t}\n\t\n\tprotected void computeGradient(double[][] block, double[] rTrain, double[] gradient) {\n\t\tfor (int i = 0; i < block.length; i++) {\n\t\t\tgradient[i] = VectorUtils.dotProduct(block[i], rTrain) / rTrain.length;\n\t\t}\n\t}\n\n\tprotected void computeGradient(int[][] index, double[][] block, double[] rTrain, double[] gradient) {\n\t\tfor (int j = 0; j < block.length; j++) {\n\t\t\tdouble[] t = block[j];\n\t\t\tint[] idx = index[j];\n\t\t\tgradient[j] = 0;\n\t\t\tfor (int i = 0; i < t.length; i++) {\n\t\t\t\tgradient[j] += rTrain[idx[i]] * t[i];\n\t\t\t}\n\t\t\tgradient[j] /= rTrain.length;\n\t\t}\n\t}\n\n\tprotected double computePenalty(double[] w, double lambda) {\n\t\treturn lambda * VectorUtils.l2norm(w);\n\t}\n\n\tprotected double computePenalty(double[][] w, double[] lambdas) {\n\t\tdouble penalty = 0;\n\t\tfor (int i = 0; i < w.length; i++) {\n\t\t\tpenalty += computePenalty(w[i], lambdas[i]);\n\t\t}\n\t\treturn penalty;\n\t}\n\t\n\tprotected DenseDesignMatrix createDesignMatrix(DenseDataset dd, List<int[]> groupList) {\n\t\tint[] attrs = dd.attrs;\n\t\tMap<Integer, Integer> attrSet = new HashMap<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tattrSet.put(attrs[j], j);\n\t\t}\n\t\tList<int[]> gList = new ArrayList<>();\n\t\tfor (int[] group : groupList) {\n\t\t\tList<Integer> list = new ArrayList<>();\n\t\t\tfor (int idx : group) {\n\t\t\t\tif (attrSet.containsKey(idx)) {\n\t\t\t\t\tlist.add(attrSet.get(idx));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (list.size() > 0) {\n\t\t\t\tint[] a = new int[list.size()];\n\t\t\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\t\t\ta[i] = list.get(i);\n\t\t\t\t}\n\t\t\t\tgList.add(a);\n\t\t\t}\n\t\t}\n\t\tint[][] groups = new int[gList.size()][];\n\t\tdouble[][][] x = new double[gList.size()][][];\n\t\tfor (int g = 0; g < groups.length; g++) {\n\t\t\tint[] group = gList.get(g);\n\t\t\tgroups[g] = group;\n\t\t\tdouble[][] v = new double[group.length][];\n\t\t\tfor (int j = 0; j < group.length; j++) {\n\t\t\t\tint idx = group[j];\n\t\t\t\tv[j] = dd.x[idx];\n\t\t\t}\n\t\t\tx[g] = v;\n\t\t}\n\t\treturn new DenseDesignMatrix(groups, x);\n\t}\n\t\n\tprotected SparseDesignMatrix createDesignMatrix(SparseDataset sd, List<int[]> groupList) {\n\t\tint[] attrs = sd.attrs;\n\t\tMap<Integer, Integer> attrSet = new HashMap<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tattrSet.put(attrs[j], j);\n\t\t}\n\t\tList<int[]> gList = new ArrayList<>();\n\t\tfor (int[] group : groupList) {\n\t\t\tList<Integer> list = new ArrayList<>();\n\t\t\tfor (int idx : group) {\n\t\t\t\tif (attrSet.containsKey(idx)) {\n\t\t\t\t\tlist.add(attrSet.get(idx));\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (list.size() > 0) {\n\t\t\t\tint[] a = new int[list.size()];\n\t\t\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\t\t\ta[i] = list.get(i);\n\t\t\t\t}\n\t\t\t\tgList.add(a);\n\t\t\t}\n\t\t}\n\t\t\n\t\tint[][] groups = new int[gList.size()][];\n\t\tint[][][] indices = new int[gList.size()][][];\n\t\tdouble[][][] values = new double[gList.size()][][];\n\t\tfor (int g = 0; g < groups.length; g++) {\n\t\t\tint[] group = gList.get(g);\n\t\t\tgroups[g] = group;\n\t\t\tint[][] idx = new int[group.length][];\n\t\t\tdouble[][] val = new double[group.length][];\n\t\t\tfor (int j = 0; j < group.length; j++) {\n\t\t\t\tint index = group[j];\n\t\t\t\tidx[j] = sd.indices[index];\n\t\t\t\tval[j] = sd.values[index];\n\t\t\t}\n\t\t\tindices[g] = idx;\n\t\t\tvalues[g] = val;\n\t\t}\n\t\treturn new SparseDesignMatrix(groups, indices, values);\n\t}\n\n\tprotected boolean doOnePassGaussian(double[][][] x, double[] tl1, boolean isFullPass, boolean[] activeSet,\n\t\t\tdouble[][] w, double[] stepSize, double[] g, double[] gradient, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < x.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\tdouble norm = Math.sqrt(StatUtils.sumSq(g, 0, beta.length));\n\t\t\tdouble lambda = tl1[k] * tk;\n\t\t\tif (norm > lambda) {\n\t\t\t\tVectorUtils.multiply(g, (1 - lambda / norm));\n\t\t\t} else {\n\t\t\t\tArrays.fill(g, 0);\n\t\t\t}\n\n\t\t\t// Update residuals\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = beta[j] - g[j];\n\t\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\t\trTrain[i] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\t\n\tprotected boolean doOnePassGaussian(int[][][] indices, double[][][] values, double[] tl1, boolean isFullPass,\n\t\t\tboolean[] activeSet, double[][] w, double[] stepSize, double[] g, double[] gradient, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = values[k];\n\t\t\tint[][] index = indices[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = tl1[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\tdouble norm = Math.sqrt(StatUtils.sumSq(g, 0, beta.length));\n\t\t\tdouble lambda = tl1[k] * tk;\n\t\t\tif (norm > lambda) {\n\t\t\t\tVectorUtils.multiply(g, (1 - lambda / norm));\n\t\t\t} else {\n\t\t\t\tVectorUtils.multiply(g, 0);\n\t\t\t}\n\t\t\t\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tint[] idx = index[j];\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = beta[j] - g[j];\n\t\t\t\tfor (int i = 0; i < t.length; i++) {\n\t\t\t\t\trTrain[idx[i]] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected boolean doOnePassBinomial(double[][][] x, double[] y, double[] tl1, boolean isFullPass, boolean[] activeSet,\n\t\t\tdouble[][] w, double[] stepSize, double[] g, double[] gradient, double[] pTrain, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < x.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[][] block = x[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\tdouble norm = Math.sqrt(StatUtils.sumSq(g, 0, beta.length));\n\t\t\tdouble lambda = tl1[k] * tk;\n\t\t\tif (norm > lambda) {\n\t\t\t\tVectorUtils.multiply(g, (1 - lambda / norm));\n\t\t\t} else {\n\t\t\t\tArrays.fill(g, 0);\n\t\t\t}\n\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tdouble[] t = block[j];\n\t\t\t\tdouble delta = g[j] - beta[j];\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tpTrain[i] += delta * t[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected boolean doOnePassBinomial(int[][][] indices, int[][] indexUnion, double[][][] values, double[] y, double[] tl1, \n\t\t\tboolean isFullPass, boolean[] activeSet, double[][] w, double[] stepSize, double[] g, double[] gradient,\n\t\t\tdouble[] pTrain, double[] rTrain) {\n\t\tboolean activeSetChanged = false;\n\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tif (!isFullPass && !activeSet[k]) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tint[][] index = indices[k];\n\t\t\tdouble[][] block = values[k];\n\t\t\tdouble[] beta = w[k];\n\t\t\tdouble tk = stepSize[k];\n\n\t\t\t// Proximal gradient method\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tg[j] = beta[j] + tk * gradient[j];\n\t\t\t}\n\n\t\t\tdouble norm = Math.sqrt(StatUtils.sumSq(g, 0, beta.length));\n\t\t\tdouble lambda = tl1[k] * tk;\n\t\t\tif (norm > lambda) {\n\t\t\t\tVectorUtils.multiply(g, (1 - lambda / norm));\n\t\t\t} else {\n\t\t\t\tVectorUtils.multiply(g, 0);\n\t\t\t}\n\t\t\t\n\t\t\t// Update predictions\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tint[] idx = index[j];\n\t\t\t\tdouble[] value = block[j];\n\t\t\t\tdouble delta = g[j] - beta[j];\n\t\t\t\tfor (int i = 0; i < value.length; i++) {\n\t\t\t\t\tpTrain[idx[i]] += delta * value[i];\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\tint[] idxUnion = indexUnion[k];\n\t\t\tfor (int idx : idxUnion) {\n\t\t\t\trTrain[idx] = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t}\n\n\t\t\t// Update weights\n\t\t\tfor (int j = 0; j < beta.length; j++) {\n\t\t\t\tbeta[j] = g[j];\n\t\t\t}\n\n\t\t\tif (isFullPass && !activeSet[k] && !ArrayUtils.isConstant(beta, 0, beta.length, 0)) {\n\t\t\t\tactiveSetChanged = true;\n\t\t\t\tactiveSet[k] = true;\n\t\t\t}\n\t\t}\n\n\t\treturn activeSetChanged;\n\t}\n\n\tprotected double findMaxLambdaGaussian(double[][][] x, double[] rTrain, double[] gradient) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(rTrain);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (double[][] block : x) {\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(rTrain, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\t\n\tprotected double findMaxLambdaGaussian(int[][][] indices, double[][][] values, double[] rTrain, double[] gradient) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(rTrain);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (int g = 0; g < values.length; g++) {\n\t\t\tint[][] index = indices[g];\n\t\t\tdouble[][] block = values[g];\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(rTrain, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\tprotected double findMaxLambdaBinomial(double[][][] x, double[] y, double[] pTrain, double[] rTrain, double[] gradient) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (double[][] block : x) {\n\t\t\tcomputeGradient(block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\t\n\tprotected double findMaxLambdaBinomial(int[][][] indices, double[][][] values, double[] y, double[] pTrain,\n\t\t\tdouble[] rTrain, double[] gradient) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (int j = 0; j < values.length; j++) {\n\t\t\tint[][] index = indices[j];\n\t\t\tdouble[][] block = values[j];\n\t\t\tcomputeGradient(index, block, rTrain, gradient);\n\t\t\tdouble t = Math.sqrt(StatUtils.sumSq(gradient, 0, block.length)) / Math.sqrt(block.length);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\t\n\tprotected GLM getGLM(int p, int[][] attrs, boolean[] selected, double[] coef, double intercept, LinkFunction link) {\n\t\tGLM glm = new GLM(p);\n\t\tint k = 0;\n\t\tdouble[] w = glm.coefficients(0);\n\t\tfor (int g = 0; g < attrs.length; g++) {\n\t\t\tif (selected[g]) {\n\t\t\t\tint[] attr = attrs[g];\n\t\t\t\tfor (int attIndex : attr) {\n\t\t\t\t\tw[attIndex] = coef[k++];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tglm.intercept[0] = intercept;\n\t\tglm.link = link;\n\t\treturn glm;\n\t}\n\t\n\tprotected GLM getGLM(int p, int[][] attrs, double[][] coef, double intercept, LinkFunction link) {\n\t\tGLM glm = new GLM(p);\n\t\tdouble[] w = glm.coefficients(0);\n\t\tfor (int g = 0; g < attrs.length; g++) {\n\t\t\tint[] attr = attrs[g];\n\t\t\tdouble[] beta = coef[g];\n\t\t\tfor (int j = 0; j < attr.length; j++) {\n\t\t\t\tw[attr[j]] = beta[j];\n\t\t\t}\n\t\t}\n\t\tglm.intercept[0] = intercept;\n\t\tglm.link = link;\n\t\treturn glm;\n\t}\n\t\n\tprotected GLM refitClassifier(int p, int[][] groups, boolean[] selected, double[][][] x, double[] y, double[][] w, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int g = 0; g < selected.length; g++) {\n\t\t\tif (selected[g]) {\n\t\t\t\tdouble[][] t = x[g];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\txList.add(t[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\n\t\tint[] attrs = new int[xNew.length];\n\t\tfor (int i = 0; i < attrs.length; i++) {\n\t\t\tattrs[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildBinaryClassifier(attrs, xNew, y, maxNumIters, 1e-8);\n\t\treturn getGLM(p, groups, selected, glm.coefficients(0), glm.intercept(0), LinkFunction.LOGIT);\n\t}\n\t\n\tprotected GLM refitClassifier(int p, int[][] groups, boolean[] selected, int[][][] indices, double[][][] values, double[] y, double[][] w, int maxNumIters) {\n\t\tList<int[]> iList = new ArrayList<>();\n\t\tList<double[]> vList = new ArrayList<>();\n\t\tfor (int g = 0; g < selected.length; g++) {\n\t\t\tif (selected[g]) {\n\t\t\t\tint[][] iBlock = indices[g];\n\t\t\t\tdouble[][] vBlock = values[g];\n\t\t\t\tfor (int j = 0; j < vBlock.length; j++) {\n\t\t\t\t\tiList.add(iBlock[j]);\n\t\t\t\t\tvList.add(vBlock[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tint[][] idxNew = new int[iList.size()][];\n\t\tfor (int i = 0; i < idxNew.length; i++) {\n\t\t\tidxNew[i] = iList.get(i);\n\t\t}\n\t\tdouble[][] valNew = new double[vList.size()][];\n\t\tfor (int i = 0; i < valNew.length; i++) {\n\t\t\tvalNew[i] = vList.get(i);\n\t\t}\n\n\t\tint[] attrs = new int[valNew.length];\n\t\tfor (int i = 0; i < attrs.length; i++) {\n\t\t\tattrs[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildBinaryClassifier(attrs, idxNew, valNew, y, maxNumIters, 1e-8);\n\t\treturn getGLM(p, groups, selected, glm.coefficients(0), glm.intercept(0), LinkFunction.LOGIT);\n\t}\n\t\n\tprotected GLM refitGaussianRegressor(int p, int[][] attrs, boolean[] selected, double[][][] x, double[] y, double[][] w, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int g = 0; g < selected.length; g++) {\n\t\t\tif (selected[g]) {\n\t\t\t\tdouble[][] t = x[g];\n\t\t\t\tfor (int j = 0; j < t.length; j++) {\n\t\t\t\t\txList.add(t[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (xList.size() == 0) {\n\t\t\tif (fitIntercept) {\n\t\t\t\tdouble intercept = StatUtils.mean(y);\n\t\t\t\tGLM glm = new GLM();\n\t\t\t\tglm.intercept[0] = intercept;\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\treturn new GLM();\n\t\t\t}\n\t\t}\n\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\n\t\tint[] attrsNew = new int[xNew.length];\n\t\tfor (int j = 0; j < attrsNew.length; j++) {\n\t\t\tattrsNew[j] = j;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildGaussianRegressor(attrsNew, xNew, y, maxNumIters, 1e-8);\n\t\treturn getGLM(p, attrs, selected, glm.coefficients(0), glm.intercept(0), LinkFunction.IDENTITY);\n\t}\n\n\tprotected GLM refitGaussianRegressor(int p, int[][] groups, boolean[] selected, int[][][] indices, double[][][] values, double[] y, double[][] w, int maxNumIters) {\n\t\tList<int[]> iList = new ArrayList<>();\n\t\tList<double[]> vList = new ArrayList<>();\n\t\tfor (int g = 0; g < selected.length; g++) {\n\t\t\tif (selected[g]) {\n\t\t\t\tint[][] iBlock = indices[g];\n\t\t\t\tdouble[][] vBlock = values[g];\n\t\t\t\tfor (int j = 0; j < vBlock.length; j++) {\n\t\t\t\t\tiList.add(iBlock[j]);\n\t\t\t\t\tvList.add(vBlock[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (vList.size() == 0) {\n\t\t\tif (fitIntercept) {\n\t\t\t\tdouble intercept = StatUtils.mean(y);\n\t\t\t\tGLM glm = new GLM();\n\t\t\t\tglm.intercept[0] = intercept;\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\treturn new GLM();\n\t\t\t}\n\t\t}\n\n\t\tint[][] idxNew = new int[iList.size()][];\n\t\tfor (int i = 0; i < idxNew.length; i++) {\n\t\t\tidxNew[i] = iList.get(i);\n\t\t}\n\t\tdouble[][] valNew = new double[vList.size()][];\n\t\tfor (int i = 0; i < valNew.length; i++) {\n\t\t\tvalNew[i] = vList.get(i);\n\t\t}\n\n\t\tint[] attrs = new int[valNew.length];\n\t\tfor (int i = 0; i < attrs.length; i++) {\n\t\t\tattrs[i] = i;\n\t\t}\n\n\t\tRidgeLearner ridgeLearner = new RidgeLearner();\n\t\tridgeLearner.setVerbose(verbose);\n\t\tridgeLearner.setEpsilon(epsilon);\n\t\tridgeLearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = ridgeLearner.buildGaussianRegressor(attrs, idxNew, valNew, y, maxNumIters, 1e-8);\n\t\treturn getGLM(p, groups, selected, glm.coefficients(0), glm.intercept(0), LinkFunction.IDENTITY);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/LassoLearner.java",
    "content": "package mltk.predictor.glm;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerWithTaskOptions;\nimport mltk.core.Attribute;\nimport mltk.core.DenseVector;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.SparseVector;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Family;\nimport mltk.predictor.LinkFunction;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for learning L1-regularized linear model via coordinate descent.\n * \n * @author Yin Lou\n * \n */\npublic class LassoLearner extends GLMLearner {\n\t\n\tstatic class Options extends LearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-m\", description = \"maximum num of iterations (default: 0)\")\n\t\tint maxIter = 0;\n\n\t\t@Argument(name = \"-l\", description = \"lambda (default: 0)\")\n\t\tdouble lambda = 0;\n\n\t}\n\n\t/**\n\t * Trains L1-regularized GLMs.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.glm.LassoLearner\n\t * -t\ttrain set path\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-m]\tmaximum num of iterations (default: 0)\n\t * [-l]\tlambda (default: 0)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(LassoLearner.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tLassoLearner learner = new LassoLearner();\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setTask(task);\n\t\tlearner.setLambda(opts.lambda);\n\t\tlearner.setMaxNumIters(opts.maxIter);\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGLM glm = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(glm, opts.outputModelPath);\n\t\t}\n\t}\n\n\tstatic class ModelStructure {\n\n\t\tboolean[] structure;\n\n\t\tModelStructure(boolean[] structure) {\n\t\t\tthis.structure = structure;\n\t\t}\n\n\t\t@Override\n\t\tpublic boolean equals(Object obj) {\n\t\t\tif (this == obj)\n\t\t\t\treturn true;\n\t\t\tif (obj == null)\n\t\t\t\treturn false;\n\t\t\tif (getClass() != obj.getClass())\n\t\t\t\treturn false;\n\t\t\tModelStructure other = (ModelStructure) obj;\n\t\t\tif (!Arrays.equals(structure, other.structure))\n\t\t\t\treturn false;\n\t\t\treturn true;\n\t\t}\n\n\t\t@Override\n\t\tpublic int hashCode() {\n\t\t\tfinal int prime = 31;\n\t\t\tint result = 1;\n\t\t\tresult = prime * result + Arrays.hashCode(structure);\n\t\t\treturn result;\n\t\t}\n\n\t}\n\n\tprotected boolean refit;\n\tprotected int numLambdas;\n\tprotected double lambda;\n\tprotected Task task;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LassoLearner() {\n\t\trefit = false;\n\t\tlambda = 0; // no regularization\n\t\tnumLambdas = -1; // no regularization path\n\t\ttask = Task.REGRESSION;\n\t}\n\n\t@Override\n\tpublic GLM build(Instances instances) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tglm = buildGaussianRegressor(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tglm = buildClassifier(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn glm;\n\t}\n\t\n\t@Override\n\tpublic GLM build(Instances trainSet, Family family) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (family) {\n\t\t\tcase GAUSSIAN:\n\t\t\t\tglm = buildGaussianRegressor(trainSet, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase BINOMIAL:\n\t\t\t\tglm = buildClassifier(trainSet, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tthrow new IllegalArgumentException(\"Unsupported family: \" + family);\n\t\t}\n\t\treturn glm;\n\t}\n\n\t/**\n\t * Builds an L1-regularized binary classifier. Each row in the input matrix x represents a feature (instead of a\n\t * data point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl1 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(x, theta, y, tl1, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t}\n\t\t\treturn refitClassifier(attrs, selected, x, y, maxNumIters);\n\t\t} else {\n\t\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L1-regularized binary classifier on sparse inputs. Each row of the input represents a feature (instead\n\t * of a data point), i.e., in column-oriented format. This procedure does not assume the data is normalized or\n\t * centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl1 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(indices, values, theta, y, tl1, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t}\n\t\t\treturn refitClassifier(attrs, selected, indices, values, y, maxNumIters);\n\t\t} else {\n\t\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\t}\n\n\t/**\n\t * Builds L1-regularized binary classifiers for a sequence of regularization parameter lambdas. Each row in the\n\t * input matrix x represents a feature (instead of a data point). Thus the input matrix is the transpose of the\n\t * row-oriented data matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized classifiers.\n\t */\n\tpublic List<GLM> buildBinaryClassifiers(int[] attrs, double[][] x, double[] y, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(x, y, pTrain, rTrain);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < numLambdas; g++) {\n\t\t\t// Coordinate descent\n\t\t\tfinal double tl1 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(x, theta, y, tl1, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitClassifier(attrs, selected, x, y, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds L1-regularized binary classifiers for a sequence of regularization parameter lambdas on sparse format.\n\t * Each row of the input represents a feature (instead of a data point), i.e., in column-oriented format. This\n\t * procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized classifiers.\n\t */\n\tpublic List<GLM> buildBinaryClassifiers(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\tdouble maxLambda = findMaxLambdaBinomial(indices, values, y, pTrain, rTrain);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < numLambdas; g++) {\n\t\t\t// Coordinate descent\n\t\t\tfinal double tl1 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(indices, values, theta, y, tl1, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(pTrain, y, w, lambda);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitClassifier(attrs, selected, indices, values, y, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds an L1-regularized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized classifer.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, boolean isSparse, int maxNumIters, double lambda) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L1-regularized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized classifer.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, int maxNumIters, double lambda) {\n\t\treturn buildClassifier(trainSet, isSparse(trainSet), maxNumIters, lambda);\n\t}\n\n\t/**\n\t * Builds L1-regularized classifiers for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized classifiers.\n\t */\n\tpublic List<GLM> buildClassifiers(Instances trainSet, boolean isSparse, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getCardinality();\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tList<GLM> glms = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters, numLambdas,\n\t\t\t\t\t\tminLambdaRatio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tboolean refit = this.refit;\n\t\t\t\tthis.refit = false; // Not supported in multiclass\n\t\t\t\t\t\t\t\t\t// classification\n\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tList<GLM> glms = new ArrayList<>();\n\t\t\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t\t\tGLM glm = new GLM(numClasses, p);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tList<GLM> binaryClassifiers = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters,\n\t\t\t\t\t\t\tnumLambdas, minLambdaRatio);\n\n\t\t\t\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers.get(l);\n\t\t\t\t\t\tGLM glm = glms.get(l);\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tthis.refit = refit;\n\n\t\t\t\treturn glms;\n\t\t\t}\n\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tList<GLM> glms = buildBinaryClassifiers(attrs, x, y, maxNumIters, numLambdas, minLambdaRatio);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tboolean refit = this.refit;\n\t\t\t\tthis.refit = false; // Not supported in multiclass\n\t\t\t\t\t\t\t\t\t// classification\n\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tList<GLM> glms = new ArrayList<>();\n\t\t\t\tfor (int i = 0; i < numLambdas; i++) {\n\t\t\t\t\tGLM glm = new GLM(numClasses, p);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tList<GLM> binaryClassifiers = buildBinaryClassifiers(attrs, x, y, maxNumIters, numLambdas,\n\t\t\t\t\t\t\tminLambdaRatio);\n\n\t\t\t\t\tfor (int l = 0; l < numLambdas; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers.get(l);\n\t\t\t\t\t\tGLM glm = glms.get(l);\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tthis.refit = refit;\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds L1-regularized classifiers for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized classifiers.\n\t */\n\tpublic List<GLM> buildClassifiers(Instances trainSet, int maxNumIters, int numLambdas, double minLambdaRatio) {\n\t\treturn buildClassifiers(trainSet, isSparse(trainSet), maxNumIters, numLambdas, minLambdaRatio);\n\t}\n\n\t/**\n\t * Builds an L1-regularized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, boolean isSparse, int maxNumIters, double lambda) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, lambda);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\n\t\t\treturn glm;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(dd.attrs, dd.x, dd.y, maxNumIters, lambda);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\t\t\treturn glm;\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L1-regularized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, int maxNumIters, double lambda) {\n\t\treturn buildGaussianRegressor(trainSet, isSparse(trainSet), maxNumIters, lambda);\n\t}\n\n\t/**\n\t * Builds an L1-regularized regressor. Each row in the input matrix x represents a feature (instead of a data\n\t * point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not assume\n\t * the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized penalized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tsq[j] = StatUtils.sumSq(x[j]);\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl1 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(x, sq, tl1, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t}\n\t\t\treturn refitGaussianRegressor(attrs, selected, x, y, maxNumIters);\n\t\t} else {\n\t\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L1-regularized regressor on sparse inputs. Each row of the input represents a feature (instead of a\n\t * data point), i.e., in column-oriented format. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L1-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[values.length];\n\t\tfor (int j = 0; j < values.length; j++) {\n\t\t\tsq[j] = StatUtils.sumSq(values[j]);\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl1 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(indices, values, sq, tl1, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\tif (refit) {\n\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t}\n\t\t\treturn refitGaussianRegressor(attrs, selected, indices, values, y, maxNumIters);\n\t\t} else {\n\t\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\t}\n\n\t/**\n\t * Builds L1-regularized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(Instances trainSet, boolean isSparse, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tList<GLM> glms = buildGaussianRegressors(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, numLambdas,\n\t\t\t\t\tminLambdaRatio);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int i = 0; i < cList.length; i++) {\n\t\t\t\t\tint attIndex = sd.attrs[i];\n\t\t\t\t\tw[attIndex] *= cList[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tList<GLM> glms = buildGaussianRegressors(dd.attrs, dd.x, dd.y, maxNumIters, numLambdas, minLambdaRatio);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t}\n\t}\n\n\t/**\n\t * Builds L1-regularized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(Instances trainSet, int maxNumIters, int numLambdas, double minLambdaRatio) {\n\t\treturn buildGaussianRegressors(trainSet, isSparse(trainSet), maxNumIters, numLambdas, minLambdaRatio);\n\t}\n\n\t/**\n\t * Builds L1-regularized regressors for a sequence of regularization parameter lambdas. Each row in the input matrix\n\t * x represents a feature (instead of a data point). Thus the input matrix is the transpose of the row-oriented data\n\t * matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(int[] attrs, double[][] x, double[] y, int maxNumIters, int numLambdas,\n\t\t\tdouble minLambdaRatio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(x[i]);\n\t\t}\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(x, rTrain);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < numLambdas; g++) {\n\t\t\tfinal double tl1 = lambda * y.length;\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(x, sq, tl1, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitGaussianRegressor(attrs, selected, x, y, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds L1-regularized regressors for a sequence of regularization parameter lambdas on sparse format. Each row of\n\t * the input represents a feature (instead of a data point), i.e., in column-oriented format. This procedure does\n\t * not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param numLambdas the number of lambdas.\n\t * @param minLambdaRatio the minimum lambda is minLambdaRatio * max lambda.\n\t * @return L1-regularized regressors.\n\t */\n\tpublic List<GLM> buildGaussianRegressors(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tint numLambdas, double minLambdaRatio) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(values[i]);\n\t\t}\n\n\t\t// Determine max lambda\n\t\tdouble maxLambda = findMaxLambdaGaussian(indices, values, y);\n\n\t\t// Dampening factor for lambda\n\t\tdouble alpha = Math.pow(minLambdaRatio, 1.0 / numLambdas);\n\n\t\t// Compute the regularization path\n\t\tList<GLM> glms = new ArrayList<>(numLambdas);\n\t\tSet<ModelStructure> structures = new HashSet<>();\n\t\tdouble lambda = maxLambda;\n\t\tfor (int g = 0; g < numLambdas; g++) {\n\t\t\tfinal double tl1 = lambda * y.length;\n\t\t\t// Coordinate descent\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(indices, values, sq, tl1, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeLassoLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlambda *= alpha;\n\t\t\tif (refit) {\n\t\t\t\tboolean[] selected = new boolean[attrs.length];\n\t\t\t\tfor (int i = 0; i < selected.length; i++) {\n\t\t\t\t\tselected[i] = w[i] != 0;\n\t\t\t\t}\n\t\t\t\tModelStructure structure = new ModelStructure(selected);\n\t\t\t\tif (!structures.contains(structure)) {\n\t\t\t\t\tGLM glm = refitGaussianRegressor(attrs, selected, indices, values, y, maxNumIters);\n\t\t\t\t\tglms.add(glm);\n\t\t\t\t\tstructures.add(structure);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tGLM glm = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t\t\tglms.add(glm);\n\t\t\t}\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\tprotected void doOnePassGaussian(double[][] x, double[] sq, final double tl1, double[] w, double[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tdouble[] v = x[j];\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble wNew = w[j] * sq[j] + VectorUtils.dotProduct(v, rTrain);\n\n\t\t\tif (Math.abs(wNew) <= tl1) {\n\t\t\t\twNew = 0;\n\t\t\t} else if (wNew > 0) {\n\t\t\t\twNew -= tl1;\n\t\t\t} else {\n\t\t\t\twNew += tl1;\n\t\t\t}\n\t\t\twNew /= sq[j];\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\trTrain[i] -= delta * v[i];\n\t\t\t}\n\t\t}\n\t}\n\t\n\tprotected void doOnePassGaussian(int[][] indices, double[][] values, double[] sq, final double tl1, double[] w,\n\t\t\tdouble[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble wNew = w[j] * sq[j];\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\twNew += rTrain[index[i]] * value[i];\n\t\t\t}\n\n\t\t\tif (Math.abs(wNew) <= tl1) {\n\t\t\t\twNew = 0;\n\t\t\t} else if (wNew > 0) {\n\t\t\t\twNew -= tl1;\n\t\t\t} else {\n\t\t\t\twNew += tl1;\n\t\t\t}\n\t\t\twNew /= sq[j];\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\trTrain[index[i]] -= delta * value[i];\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected void doOnePassBinomial(double[][] x, double[] theta, double[] y, final double tl1, double[] w, double[] pTrain,\n\t\t\tdouble[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[] v = x[j];\n\t\t\tdouble eta = VectorUtils.dotProduct(rTrain, v);\n\n\t\t\tdouble newW = w[j] + eta / theta[j];\n\t\t\tdouble t = tl1 / theta[j];\n\t\t\tif (newW > t) {\n\t\t\t\tnewW -= t;\n\t\t\t} else if (newW < -t) {\n\t\t\t\tnewW += t;\n\t\t\t} else {\n\t\t\t\tnewW = 0;\n\t\t\t}\n\n\t\t\tdouble delta = newW - w[j];\n\t\t\tw[j] = newW;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < pTrain.length; i++) {\n\t\t\t\tpTrain[i] += delta * v[i];\n\t\t\t\trTrain[i] = OptimUtils.getPseudoResidual(pTrain[i], y[i]);\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected void doOnePassBinomial(int[][] indices, double[][] values, double[] theta, double[] y, final double tl1, double[] w,\n\t\t\tdouble[] pTrain, double[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble eta = 0;\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\teta += rTrain[idx] * value[i];\n\t\t\t}\n\n\t\t\tdouble newW = w[j] + eta / theta[j];\n\t\t\tdouble t = tl1 / theta[j];\n\t\t\tif (newW > t) {\n\t\t\t\tnewW -= t;\n\t\t\t} else if (newW < -t) {\n\t\t\t\tnewW += t;\n\t\t\t} else {\n\t\t\t\tnewW = 0;\n\t\t\t}\n\n\t\t\tdouble delta = newW - w[j];\n\t\t\tw[j] = newW;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\tpTrain[idx] += delta * value[i];\n\t\t\t\trTrain[idx] = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected double findMaxLambdaGaussian(double[][] x, double[] y) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(y);\n\t\t}\n\t\t// Determine max lambda\n\t\tdouble maxLambda = 0;\n\t\tfor (double[] col : x) {\n\t\t\tdouble dot = Math.abs(VectorUtils.dotProduct(col, y));\n\t\t\tif (dot > maxLambda) {\n\t\t\t\tmaxLambda = dot;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(y, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\t\n\tprotected double findMaxLambdaGaussian(int[][] indices, double[][] values, double[] y) {\n\t\tdouble mean = 0;\n\t\tif (fitIntercept) {\n\t\t\tmean = OptimUtils.fitIntercept(y);\n\t\t}\n\n\t\tDenseVector v = new DenseVector(y);\n\t\t// Determine max lambda\n\t\tdouble maxLambda = 0;\n\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\tint[] index = indices[i];\n\t\t\tdouble[] value = values[i];\n\t\t\tdouble dot = Math.abs(VectorUtils.dotProduct(new SparseVector(index, value), v));\n\t\t\tif (dot > maxLambda) {\n\t\t\t\tmaxLambda = dot;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tif (fitIntercept) {\n\t\t\tVectorUtils.add(y, mean);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\tprotected double findMaxLambdaBinomial(double[][] x, double[] y, double[] pTrain, double[] rTrain) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (double[] col : x) {\n\t\t\tdouble eta = 0;\n\t\t\tfor (int i = 0; i < col.length; i++) {\n\t\t\t\tdouble r = OptimUtils.getPseudoResidual(pTrain[i], y[i]);\n\t\t\t\tr *= col[i];\n\t\t\t\teta += r;\n\t\t\t}\n\n\t\t\tdouble t = Math.abs(eta);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\tprotected double findMaxLambdaBinomial(int[][] indices, double[][] values, double[] y, double[] pTrain, double[] rTrain) {\n\t\tif (fitIntercept) {\n\t\t\tOptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t}\n\t\tdouble maxLambda = 0;\n\t\tfor (int k = 0; k < values.length; k++) {\n\t\t\tdouble eta = 0;\n\t\t\tint[] index = indices[k];\n\t\t\tdouble[] value = values[k];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\tdouble r = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t\tr *= value[i];\n\t\t\t\teta += r;\n\t\t\t}\n\t\t\tdouble t = Math.abs(eta);\n\t\t\tif (t > maxLambda) {\n\t\t\t\tmaxLambda = t;\n\t\t\t}\n\t\t}\n\t\tmaxLambda /= y.length;\n\t\tif (fitIntercept) {\n\t\t\tArrays.fill(pTrain, 0);\n\t\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\t\t}\n\t\treturn maxLambda;\n\t}\n\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\n\t/**\n\t * Returns the number of lambdas.\n\t * \n\t * @return the number of lambdas.\n\t */\n\tpublic int getNumLambdas() {\n\t\treturn numLambdas;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Returns {@code true} if we refit the model.\n\t * \n\t * @return {@code true} if we refit the model.\n\t */\n\tpublic boolean refit() {\n\t\treturn refit;\n\t}\n\n\t/**\n\t * Sets whether we refit the model.\n\t * \n\t * @param refit {@code true} if we refit the model.\n\t */\n\tpublic void refit(boolean refit) {\n\t\tthis.refit = refit;\n\t}\n\n\tprotected GLM refitGaussianRegressor(int[] attrs, boolean[] selected, double[][] x, double[] y, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\txList.add(x[j]);\n\t\t\t}\n\t\t}\n\t\tif (xList.size() == 0) {\n\t\t\tGLM glm = new GLM(0);\n\t\t\tif (fitIntercept) {\n\t\t\t\tglm.intercept[0] = StatUtils.mean(y);\n\t\t\t}\n\t\t\treturn glm;\n\t\t}\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\t\tint[] attrsNew = new int[xNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setEpsilon(epsilon);\n\t\tlearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = learner.buildGaussianRegressor(attrsNew, xNew, y, maxNumIters, 1e-8);\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tw[j] = coef[k++];\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, glm.intercept(0), LinkFunction.IDENTITY);\n\t}\n\t\n\n\tprotected GLM refitGaussianRegressor(int[] attrs, boolean[] selected, int[][] indices, double[][] values, double[] y, int maxNumIters) {\n\t\tList<int[]> indicesList = new ArrayList<>();\n\t\tList<double[]> valuesList = new ArrayList<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tindicesList.add(indices[j]);\n\t\t\t\tvaluesList.add(values[j]);\n\t\t\t}\n\t\t}\n\t\tif (indicesList.size() == 0) {\n\t\t\tGLM glm = new GLM(0);\n\t\t\tif (fitIntercept) {\n\t\t\t\tglm.intercept[0] = StatUtils.mean(y);\n\t\t\t}\n\t\t\treturn glm;\n\t\t}\n\t\tint[][] indicesNew = new int[indicesList.size()][];\n\t\tfor (int i = 0; i < indicesNew.length; i++) {\n\t\t\tindicesNew[i] = indicesList.get(i);\n\t\t}\n\t\tdouble[][] valuesNew = new double[valuesList.size()][];\n\t\tfor (int i = 0; i < indicesNew.length; i++) {\n\t\t\tvaluesNew[i] = valuesList.get(i);\n\t\t}\n\t\tint[] attrsNew = new int[indicesNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setEpsilon(epsilon);\n\t\tlearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = learner.buildGaussianRegressor(attrsNew, indicesNew, valuesNew, y, maxNumIters, 1e-8);\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tw[j] = coef[k++];\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, glm.intercept(0), LinkFunction.IDENTITY);\n\t}\n\n\tprotected GLM refitClassifier(int[] attrs, boolean[] selected, double[][] x, double[] y, int maxNumIters) {\n\t\tList<double[]> xList = new ArrayList<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\txList.add(x[j]);\n\t\t\t}\n\t\t}\n\n\t\tdouble[][] xNew = new double[xList.size()][];\n\t\tfor (int i = 0; i < xNew.length; i++) {\n\t\t\txNew[i] = xList.get(i);\n\t\t}\n\t\tint[] attrsNew = new int[xNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setEpsilon(epsilon);\n\t\tlearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = learner.buildBinaryClassifier(attrsNew, xNew, y, maxNumIters, 1e-8);\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tw[j] = coef[k++];\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, glm.intercept(0), LinkFunction.LOGIT);\n\t}\n\n\tprotected GLM refitClassifier(int[] attrs, boolean[] selected, int[][] indices, double[][] values, double[] y, int maxNumIters) {\n\t\tList<int[]> indicesList = new ArrayList<>();\n\t\tList<double[]> valuesList = new ArrayList<>();\n\t\tfor (int j = 0; j < attrs.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tindicesList.add(indices[j]);\n\t\t\t\tvaluesList.add(values[j]);\n\t\t\t}\n\t\t}\n\n\t\tint[][] indicesNew = new int[indicesList.size()][];\n\t\tfor (int i = 0; i < indicesNew.length; i++) {\n\t\t\tindicesNew[i] = indicesList.get(i);\n\t\t}\n\t\tdouble[][] valuesNew = new double[valuesList.size()][];\n\t\tfor (int i = 0; i < indicesNew.length; i++) {\n\t\t\tvaluesNew[i] = valuesList.get(i);\n\t\t}\n\t\tint[] attrsNew = new int[indicesNew.length];\n\t\tfor (int i = 0; i < attrsNew.length; i++) {\n\t\t\tattrsNew[i] = i;\n\t\t}\n\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setVerbose(verbose);\n\t\tlearner.setEpsilon(epsilon);\n\t\tlearner.fitIntercept(fitIntercept);\n\t\t// A ridge regression with very small regularization parameter\n\t\t// This often improves stability a lot\n\t\tGLM glm = learner.buildBinaryClassifier(attrsNew, indicesNew, valuesNew, y, maxNumIters, 1e-8);\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble[] coef = glm.coefficients(0);\n\t\tint k = 0;\n\t\tfor (int j = 0; j < w.length; j++) {\n\t\t\tif (selected[j]) {\n\t\t\t\tw[j] = coef[k++];\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, glm.intercept(0), LinkFunction.LOGIT);\n\t}\n\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\n\t/**\n\t * Sets the number of lambdas.\n\t * \n\t * @param numLambdas the number of lambdas.\n\t */\n\tpublic void setNumLambdas(int numLambdas) {\n\t\tthis.numLambdas = numLambdas;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/RidgeLearner.java",
    "content": "package mltk.predictor.glm;\n\nimport java.util.Arrays;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerWithTaskOptions;\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Family;\nimport mltk.predictor.LinkFunction;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.StatUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for learning L2-regularized linear model via coordinate descent.\n * \n * @author Yin Lou\n * \n */\npublic class RidgeLearner extends GLMLearner {\n\n\tstatic class Options extends LearnerWithTaskOptions {\n\n\t\t@Argument(name = \"-m\", description = \"maximum num of iterations (default: 0)\")\n\t\tint maxIter = 0;\n\n\t\t@Argument(name = \"-l\", description = \"lambda (default: 0)\")\n\t\tdouble lambda = 0;\n\n\t}\n\n\t/**\n\t * Trains L2-regularized GLMs.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.glm.RidgeLearner\n\t * -t\ttrain set path\n\t * [-g]\ttask between classification (c) and regression (r) (default: r)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-m]\tmaximum num of iterations (default: 0)\n\t * [-l]\tlambda (default: 0)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(RidgeLearner.class, opts);\n\t\tTask task = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\ttask = Task.get(opts.task);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tRidgeLearner learner = new RidgeLearner();\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setTask(task);\n\t\tlearner.setLambda(opts.lambda);\n\t\tlearner.setMaxNumIters(opts.maxIter);\n\n\t\tlong start = System.currentTimeMillis();\n\t\tGLM glm = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(glm, opts.outputModelPath);\n\t\t}\n\t}\n\n\tprotected double lambda;\n\tprotected Task task;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic RidgeLearner() {\n\t\tlambda = 0; // no regularization\n\t\ttask = Task.REGRESSION;\n\t}\n\n\t@Override\n\tpublic GLM build(Instances instances) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (task) {\n\t\t\tcase REGRESSION:\n\t\t\t\tglm = buildGaussianRegressor(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase CLASSIFICATION:\n\t\t\t\tglm = buildClassifier(instances, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn glm;\n\t}\n\t\n\t@Override\n\tpublic GLM build(Instances trainSet, Family family) {\n\t\tGLM glm = null;\n\t\tif (maxNumIters < 0) {\n\t\t\tmaxNumIters = 20;\n\t\t}\n\t\tswitch (family) {\n\t\t\tcase GAUSSIAN:\n\t\t\t\tglm = buildGaussianRegressor(trainSet, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tcase BINOMIAL:\n\t\t\t\tglm = buildClassifier(trainSet, maxNumIters, lambda);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tthrow new IllegalArgumentException(\"Unsupported family: \" + family);\n\t\t}\n\t\treturn glm;\n\t}\n\n\t/**\n\t * Builds an L2-regularized binary classifier. Each row in the input matrix x represents a feature (instead of a\n\t * data point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not\n\t * assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized binary classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\t// Coordinate gradient descent\n\t\tfinal double tl2 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(x, theta, y, tl2, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t}\n\n\t/**\n\t * Builds an L2-regularized binary classifier on sparse inputs. Each row of the input represents a feature (instead\n\t * of a data point), i.e., in column-oriented format. This procedure does not assume the data is normalized or\n\t * centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized classifier.\n\t */\n\tpublic GLM buildBinaryClassifier(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\t// Coordinate gradient descent\n\t\tfinal double tl2 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t}\n\n\t\t\tdoOnePassBinomial(indices, values, theta, y, tl2, w, pTrain, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t}\n\n\t/**\n\t * Builds L2-regularized binary classifiers for a sequence of regularization parameter lambdas. Each row in the\n\t * input matrix x represents a feature (instead of a data point). Thus the input matrix is the transpose of the\n\t * row-oriented data matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized classifiers.\n\t */\n\tpublic GLM[] buildBinaryClassifiers(int[] attrs, double[][] x, double[] y, int maxNumIters, double[] lambdas) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(x[i]) / 4;\n\t\t}\n\n\t\tGLM[] glms = new GLM[lambdas.length];\n\t\tArrays.sort(lambdas);\n\n\t\tfor (int g = 0; g < glms.length; g++) {\n\t\t\tdouble lambda = lambdas[g];\n\t\t\t// Coordinate gradient descent\n\t\t\tfinal double tl2 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(x, theta, y, tl2, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds L2-regularized binary classifiers for a sequence of regularization parameter lambdas on sparse format.\n\t * Each row of the input represents a feature (instead of a data point), i.e., in column-oriented format. This\n\t * procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized classifiers.\n\t */\n\tpublic GLM[] buildBinaryClassifiers(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble[] lambdas) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tdouble[] pTrain = new double[y.length];\n\t\tdouble[] rTrain = new double[y.length];\n\t\tOptimUtils.computePseudoResidual(pTrain, y, rTrain);\n\n\t\t// Calculate theta's\n\t\tdouble[] theta = new double[values.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\ttheta[i] = StatUtils.sumSq(values[i]) / 4;\n\t\t}\n\n\t\tGLM[] glms = new GLM[lambdas.length];\n\t\tArrays.sort(lambdas);\n\n\t\tfor (int g = 0; g < glms.length; g++) {\n\t\t\tdouble lambda = lambdas[g];\n\n\t\t\t// Coordinate gradient descent\n\t\t\tfinal double tl2 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(pTrain, rTrain, y);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassBinomial(indices, values, theta, y, tl2, w, pTrain, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(pTrain, y, w, lambda);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.LOGIT);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds an L2-regularized binary classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized binary classifier.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, boolean isSparse, int maxNumIters, double lambda) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getStates().length;\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, indices, values, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM glm = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda);\n\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : attrs[attrs.length - 1] + 1;\n\t\t\t\tGLM glm = new GLM(numClasses, p);\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM binaryClassifier = buildBinaryClassifier(attrs, x, y, maxNumIters, lambda);\n\n\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t}\n\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t}\n\n\t\t\t\treturn glm;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L2-regularized classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized classifier.\n\t */\n\tpublic GLM buildClassifier(Instances trainSet, int maxNumIters, double lambda) {\n\t\treturn buildClassifier(trainSet, isSparse(trainSet), maxNumIters, lambda);\n\t}\n\n\t/**\n\t * Builds L2-regularized classifiers for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized binary classifiers.\n\t */\n\tpublic GLM[] buildClassifiers(Instances trainSet, boolean isSparse, int maxNumIters, double[] lambdas) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tint numClasses = clazz.getStates().length;\n\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tint[] attrs = sd.attrs;\n\t\t\tint[][] indices = sd.indices;\n\t\t\tdouble[][] values = sd.values;\n\t\t\tdouble[] y = new double[sd.y.length];\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM[] glms = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters, lambdas);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : (StatUtils.max(attrs) + 1);\n\t\t\t\tGLM[] glms = new GLM[lambdas.length];\n\t\t\t\tfor (int i = 0; i < glms.length; i++) {\n\t\t\t\t\tglms[i] = new GLM(numClasses, p);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) sd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM[] binaryClassifiers = buildBinaryClassifiers(attrs, indices, values, y, maxNumIters, lambdas);\n\n\t\t\t\t\tfor (int l = 0; l < glms.length; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers[l];\n\t\t\t\t\t\tGLM glm = glms[l];\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tint[] attrs = dd.attrs;\n\t\t\tdouble[][] x = dd.x;\n\t\t\tdouble[] y = new double[dd.y.length];\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tif (numClasses == 2) {\n\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\ty[i] = label == 0 ? 1 : 0;\n\t\t\t\t}\n\n\t\t\t\tGLM[] glms = buildBinaryClassifiers(attrs, x, y, maxNumIters, lambdas);\n\n\t\t\t\tfor (GLM glm : glms) {\n\t\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t} else {\n\t\t\t\tint p = attrs.length == 0 ? 0 : attrs[attrs.length - 1] + 1;\n\t\t\t\tGLM[] glms = new GLM[lambdas.length];\n\t\t\t\tfor (int i = 0; i < glms.length; i++) {\n\t\t\t\t\tglms[i] = new GLM(numClasses, p);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// One-vs-the-rest\n\t\t\t\t\tfor (int i = 0; i < y.length; i++) {\n\t\t\t\t\t\tint label = (int) dd.y[i];\n\t\t\t\t\t\ty[i] = label == k ? 1 : 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tGLM[] binaryClassifiers = buildBinaryClassifiers(attrs, x, y, maxNumIters, lambdas);\n\n\t\t\t\t\tfor (int l = 0; l < glms.length; l++) {\n\t\t\t\t\t\tGLM binaryClassifier = binaryClassifiers[l];\n\t\t\t\t\t\tGLM glm = glms[l];\n\t\t\t\t\t\tdouble[] w = binaryClassifier.w[0];\n\t\t\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\t\t\tint attIndex = attrs[j];\n\t\t\t\t\t\t\tglm.w[k][attIndex] = w[attIndex] * cList[j];\n\t\t\t\t\t\t}\n\t\t\t\t\t\tglm.intercept[k] = binaryClassifier.intercept[0];\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn glms;\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Builds L2-regularized classifiers for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized binary classifiers.\n\t */\n\tpublic GLM[] buildClassifiers(Instances trainSet, int maxNumIters, double[] lambdas) {\n\t\treturn buildClassifiers(trainSet, isSparse(trainSet), maxNumIters, lambdas);\n\t}\n\n\t/**\n\t * Builds an L2 regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, boolean isSparse, int maxNumIters, double lambda) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, lambda);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\n\t\t\treturn glm;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM glm = buildGaussianRegressor(dd.attrs, dd.x, dd.y, maxNumIters, lambda);\n\n\t\t\tdouble[] w = glm.w[0];\n\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t}\n\t\t\treturn glm;\n\t\t}\n\t}\n\n\t/**\n\t * Builds an L2-regularized regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(Instances trainSet, int maxNumIters, double lambda) {\n\t\treturn buildGaussianRegressor(trainSet, isSparse(trainSet), maxNumIters, lambda);\n\t}\n\n\t/**\n\t * Builds an L2-regularized regressor. Each row in the input matrix x represents a feature (instead of a data\n\t * point). Thus the input matrix is the transpose of the row-oriented data matrix. This procedure does not assume\n\t * the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, double[][] x, double[] y, int maxNumIters, double lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(x[i]);\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl2 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(x, sq, tl2, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t}\n\n\t/**\n\t * Builds an L2-regularized regressor on sparse inputs. Each row of the input represents a feature (instead of a\n\t * data point), i.e., in column-oriented format. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambda the lambda.\n\t * @return an L2-regularized regressor.\n\t */\n\tpublic GLM buildGaussianRegressor(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble lambda) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\t// Initialize residuals\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[attrs.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(values[i]);\n\t\t}\n\n\t\t// Coordinate descent\n\t\tfinal double tl2 = lambda * y.length;\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\n\t\t\tif (fitIntercept) {\n\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t}\n\n\t\t\tdoOnePassGaussian(indices, values, sq, tl2, w, rTrain);\n\n\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t}\n\n\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\treturn GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t}\n\n\t/**\n\t * Builds L2-regularized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param isSparse {@code true} if the training set is treated as sparse.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(Instances trainSet, boolean isSparse, int maxNumIters, double[] lambdas) {\n\t\tif (isSparse) {\n\t\t\tSparseDataset sd = getSparseDataset(trainSet, true);\n\t\t\tdouble[] cList = sd.cList;\n\n\t\t\tGLM[] glms = buildGaussianRegressors(sd.attrs, sd.indices, sd.values, sd.y, maxNumIters, lambdas);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = sd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t} else {\n\t\t\tDenseDataset dd = getDenseDataset(trainSet, true);\n\t\t\tdouble[] cList = dd.cList;\n\n\t\t\tGLM[] glms = buildGaussianRegressors(dd.attrs, dd.x, dd.y, maxNumIters, lambdas);\n\n\t\t\tfor (GLM glm : glms) {\n\t\t\t\tdouble[] w = glm.w[0];\n\t\t\t\tfor (int j = 0; j < cList.length; j++) {\n\t\t\t\t\tint attIndex = dd.attrs[j];\n\t\t\t\t\tw[attIndex] *= cList[j];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn glms;\n\t\t}\n\t}\n\n\t/**\n\t * Builds L2-regularized regressors for a sequence of regularization parameter lambdas.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(Instances trainSet, int maxNumIters, double[] lambdas) {\n\t\treturn buildGaussianRegressors(trainSet, isSparse(trainSet), maxNumIters, lambdas);\n\t}\n\n\t/**\n\t * Builds L2-regularized regressors for a sequence of regularization parameter lambdas on dense inputs. Each row in\n\t * the input matrix x represents a feature (instead of a data point). Thus the input matrix is the transpose of the\n\t * row-oriented data matrix. This procedure does not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param x the inputs.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(int[] attrs, double[][] x, double[] y, int maxNumIters, double[] lambdas) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tGLM[] glms = new GLM[lambdas.length];\n\t\tArrays.sort(lambdas);\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[x.length];\n\t\tfor (int i = 0; i < x.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(x[i]);\n\t\t}\n\n\t\t// Compute the regularization path\n\t\tfor (int g = 0; g < glms.length; g++) {\n\t\t\tdouble lambda = lambdas[g];\n\n\t\t\t// Coordinate descent\n\t\t\tfinal double tl2 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(x, sq, tl2, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\t/**\n\t * Builds L2-regularized regressors for a sequence of regularization parameter lambdas on sparse inputs. Each row of\n\t * the input represents a feature (instead of a data point), i.e., in column-oriented format. This procedure does\n\t * not assume the data is normalized or centered.\n\t * \n\t * @param attrs the attribute list.\n\t * @param indices the indices.\n\t * @param values the values.\n\t * @param y the targets.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @param lambdas the lambdas array.\n\t * @return L2-regularized regressors.\n\t */\n\tpublic GLM[] buildGaussianRegressors(int[] attrs, int[][] indices, double[][] values, double[] y, int maxNumIters,\n\t\t\tdouble[] lambdas) {\n\t\tdouble[] w = new double[attrs.length];\n\t\tdouble intercept = 0;\n\n\t\tGLM[] glms = new GLM[lambdas.length];\n\t\tArrays.sort(lambdas);\n\n\t\t// Backup targets\n\t\tdouble[] rTrain = new double[y.length];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = y[i];\n\t\t}\n\n\t\t// Calculate sum of squares\n\t\tdouble[] sq = new double[attrs.length];\n\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\tsq[i] = StatUtils.sumSq(values[i]);\n\t\t}\n\n\t\t// Compute the regularization path\n\t\tfor (int g = 0; g < glms.length; g++) {\n\t\t\tdouble lambda = lambdas[g];\n\n\t\t\t// Coordinate descent\n\t\t\tfinal double tl2 = lambda * y.length;\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\tdouble prevLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (fitIntercept) {\n\t\t\t\t\tintercept += OptimUtils.fitIntercept(rTrain);\n\t\t\t\t}\n\n\t\t\t\tdoOnePassGaussian(indices, values, sq, tl2, w, rTrain);\n\n\t\t\t\tdouble currLoss = GLMOptimUtils.computeRidgeLoss(rTrain, w, lambda);\n\t\t\t\t\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + \" \" + currLoss);\n\t\t\t\t}\n\n\t\t\t\tif (OptimUtils.isConverged(prevLoss, currLoss, epsilon)) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tglms[g] = GLMOptimUtils.getGLM(attrs, w, intercept, LinkFunction.IDENTITY);\n\t\t}\n\n\t\treturn glms;\n\t}\n\n\tprotected void doOnePassGaussian(double[][] x, double[] sq, final double tl2, double[] w, double[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tdouble[] v = x[j];\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble eta = VectorUtils.dotProduct(rTrain, v);\n\t\t\tdouble wNew = (w[j] * sq[j] + eta) / (sq[j] + tl2);\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\trTrain[i] -= delta * v[i];\n\t\t\t}\n\t\t}\n\t}\n\t\n\tprotected void doOnePassGaussian(int[][] indices, double[][] values, double[] sq, final double tl2, double[] w,\n\t\t\tdouble[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\t// Calculate weight updates using naive updates\n\t\t\tdouble wNew = w[j] * sq[j];\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\twNew += rTrain[index[i]] * value[i];\n\t\t\t}\n\t\t\twNew /= (sq[j] + tl2);\n\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\trTrain[index[i]] -= delta * value[i];\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected void doOnePassBinomial(double[][] x, double[] theta, double[] y, final double tl2, double[] w, double[] pTrain,\n\t\t\tdouble[] rTrain) {\n\t\tfor (int j = 0; j < x.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble[] v = x[j];\n\t\t\tdouble eta = VectorUtils.dotProduct(rTrain, v);\n\n\t\t\tdouble wNew = (w[j] * theta[j] + eta) / (theta[j] + tl2);\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < pTrain.length; i++) {\n\t\t\t\tpTrain[i] += delta * v[i];\n\t\t\t\trTrain[i] = OptimUtils.getPseudoResidual(pTrain[i], y[i]);\n\t\t\t}\n\t\t}\n\t}\n\t\n\tprotected void doOnePassBinomial(int[][] indices, double[][] values, double[] theta, double[] y, final double tl2, double[] w,\n\t\t\tdouble[] pTrain, double[] rTrain) {\n\t\tfor (int j = 0; j < indices.length; j++) {\n\t\t\tif (Math.abs(theta[j]) <= MathUtils.EPSILON) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tdouble eta = 0;\n\t\t\tint[] index = indices[j];\n\t\t\tdouble[] value = values[j];\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\teta += rTrain[idx] * value[i];\n\t\t\t}\n\n\t\t\tdouble wNew = (w[j] * theta[j] + eta) / (theta[j] + tl2);\n\t\t\tdouble delta = wNew - w[j];\n\t\t\tw[j] = wNew;\n\n\t\t\t// Update predictions\n\t\t\tfor (int i = 0; i < index.length; i++) {\n\t\t\t\tint idx = index[i];\n\t\t\t\tpTrain[idx] += delta * value[i];\n\t\t\t\trTrain[idx] = OptimUtils.getPseudoResidual(pTrain[idx], y[idx]);\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Returns the lambda.\n\t * \n\t * @return the lambda.\n\t */\n\tpublic double getLambda() {\n\t\treturn lambda;\n\t}\n\n\t/**\n\t * Returns the task of this learner.\n\t * \n\t * @return the task of this learner.\n\t */\n\tpublic Task getTask() {\n\t\treturn task;\n\t}\n\n\t/**\n\t * Sets the lambda.\n\t * \n\t * @param lambda the lambda.\n\t */\n\tpublic void setLambda(double lambda) {\n\t\tthis.lambda = lambda;\n\t}\n\n\t/**\n\t * Sets the task of this learner.\n\t * \n\t * @param task the task of this learner.\n\t */\n\tpublic void setTask(Task task) {\n\t\tthis.task = task;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/glm/package-info.java",
    "content": "/**\n * Provides algorithms for fitting generalized linear models (GLMs).\n */\npackage mltk.predictor.glm;"
  },
  {
    "path": "src/main/java/mltk/predictor/io/PredictorReader.java",
    "content": "package mltk.predictor.io;\n\nimport java.io.BufferedReader;\nimport java.io.FileReader;\n\nimport mltk.predictor.Predictor;\n\n/**\n * Class for reading predictors.\n * \n * @author Yin Lou\n * \n */\npublic class PredictorReader {\n\n\t/**\n\t * Reads a predictor. The caller is responsible for converting the predictor to correct type.\n\t * \n\t * @param path the file path for the predictor.\n\t * @return the parsed predictor.\n\t * @throws Exception\n\t */\n\tpublic static Predictor read(String path) throws Exception {\n\t\tBufferedReader in = new BufferedReader(new FileReader(path));\n\t\tString line = in.readLine();\n\t\tString predictorName = line.substring(1, line.length() - 1).split(\": \")[1];\n\t\tClass<?> clazz = Class.forName(predictorName);\n\t\tPredictor predictor = (Predictor) clazz.getDeclaredConstructor().newInstance();\n\t\tpredictor.read(in);\n\t\tin.close();\n\t\treturn predictor;\n\t}\n\n\t/**\n\t * Reads a predictor. The caller is responsible for providing the correct predictor type.\n\t * \n\t * @param path the file path for the predictor.\n\t * @param clazz the class of the predictor.\n\t * @param <T> the type of the predictor class.\n\t * @return the parsed predictor.\n\t * @throws Exception\n\t */\n\tpublic static <T extends Predictor> T read(String path, Class<T> clazz) throws Exception {\n\t\tPredictor predictor = read(path);\n\t\treturn clazz.cast(predictor);\n\t}\n\n\t/**\n\t * Reads a predictor from an input reader. The caller is responsible for converting the predictor to correct type.\n\t * \n\t * @param in the input reader.\n\t * @return the parsed predictor.\n\t * @throws Exception\n\t */\n\tpublic static Predictor read(BufferedReader in) throws Exception {\n\t\tString line = in.readLine();\n\t\tString predictorName = line.substring(1, line.length() - 1).split(\": \")[1];\n\t\tClass<?> clazz = Class.forName(predictorName);\n\t\tPredictor predictor = (Predictor) clazz.getDeclaredConstructor().newInstance();\n\t\tpredictor.read(in);\n\t\treturn predictor;\n\t}\n\t\n\t/**\n\t * Reads a predictor from an input reader. The caller is responsible for providing the correct predictor type.\n\t * \n\t * @param in the input reader.\n\t * @param clazz the class of the predictor.\n\t * @param <T> the type of the predictor class.\n\t * @return the parsed predictor.\n\t * @throws Exception\n\t */\n\tpublic static <T extends Predictor> T read(BufferedReader in, Class<T> clazz) throws Exception {\n\t\tPredictor predictor = read(in);\n\t\treturn clazz.cast(predictor);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/io/PredictorWriter.java",
    "content": "package mltk.predictor.io;\n\nimport java.io.PrintWriter;\n\nimport mltk.predictor.Predictor;\n\n/**\n * Class for writing predictors.\n * \n * @author Yin Lou\n * \n */\npublic class PredictorWriter {\n\n\t/**\n\t * Writes a predictor to file.\n\t * \n\t * @param predictor the predictor to write.\n\t * @param path the file path.\n\t * @throws Exception\n\t */\n\tpublic static void write(Predictor predictor, String path) throws Exception {\n\t\tPrintWriter out = new PrintWriter(path);\n\t\tpredictor.write(out);\n\t\tout.flush();\n\t\tout.close();\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/io/package-info.java",
    "content": "/**\n * Provides classes for reading and writing predictors.\n */\npackage mltk.predictor.io;"
  },
  {
    "path": "src/main/java/mltk/predictor/package-info.java",
    "content": "/**\n * Provides interfaces and classes for predictors.\n */\npackage mltk.predictor;"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/DecisionTable.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.Arrays;\n\nimport mltk.core.Instance;\nimport mltk.util.ArrayUtils;\nimport mltk.util.VectorUtils;\n\n/**\n * Class for decision tables.\n * \n * @author Yin Lou\n *\n */\npublic class DecisionTable implements RTree {\n\n\tprotected int[] attIndices;\n\tprotected double[] splits;\n\tprotected long[] predIndices;\n\tprotected double[] predValues;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic DecisionTable() {\n\t\t\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param attIndices the attribute indices.\n\t * @param splits the splits.\n\t * @param predIndices the prediction indices.\n\t * @param predValues the prediction values.\n\t */\n\tpublic DecisionTable(int[] attIndices, double[] splits,\n\t\t\tlong[] predIndices, double[] predValues) {\n\t\tthis.attIndices = attIndices;\n\t\tthis.splits = splits;\n\t\tthis.predIndices = predIndices;\n\t\tthis.predValues = predValues;\n\t}\n\t\n\t/**\n\t * Returns the attribute indices in this tree.\n\t * \n\t * @return the attribute indices in this tree.\n\t */\n\tpublic int[] getAttributeIndices() {\n\t\treturn attIndices;\n\t}\n\t\n\t/**\n\t * Returns the splits in this tree.\n\t * \n\t * @return the splits in this tree.\n\t */\n\tpublic double[] getSplits() {\n\t\treturn splits;\n\t}\n\t\n\t@Override\n\tpublic void multiply(double c) {\n\t\tVectorUtils.multiply(predValues, c);\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tin.readLine();\n\t\tattIndices = ArrayUtils.parseIntArray(in.readLine());\n\t\tin.readLine();\n\t\tsplits = ArrayUtils.parseDoubleArray(in.readLine());\n\t\tin.readLine();\n\t\tpredIndices = ArrayUtils.parseLongArray(in.readLine());\n\t\tin.readLine();\n\t\tpredValues = ArrayUtils.parseDoubleArray(in.readLine());\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Attributes: \" + attIndices.length);\n\t\tout.println(Arrays.toString(attIndices));\n\t\tout.println(\"Splits: \" + splits.length);\n\t\tout.println(Arrays.toString(splits));\n\t\tout.println(\"Prediction Indices: \" + predIndices.length);\n\t\tout.println(Arrays.toString(predIndices));\n\t\tout.println(\"Prediction Values: \" + predValues.length);\n\t\tout.println(Arrays.toString(predValues));\n\t}\n\n\t@Override\n\tpublic DecisionTable copy() {\n\t\tint[] attIndicesCopy = Arrays.copyOf(attIndices, attIndices.length);\n\t\tdouble[] splitsCopy = Arrays.copyOf(splits, splits.length);\n\t\tlong[] predIndicesCopy = Arrays.copyOf(predIndices, predIndices.length);\n\t\tdouble[] predValuesCopy = Arrays.copyOf(predValues, predValues.length);\n\t\treturn new DecisionTable(attIndicesCopy, splitsCopy, predIndicesCopy, predValuesCopy);\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\tlong predIdx = 0L;\n\t\tfor (int j = 0; j < attIndices.length; j++) {\n\t\t\tint attIndex = attIndices[j];\n\t\t\tdouble split = splits[j];\n\t\t\tif (instance.getValue(attIndex) <= split) {\n\t\t\t\tpredIdx = (predIdx << 1) | 1L;\n\t\t\t} else {\n\t\t\t\tpredIdx <<= 1;\n\t\t\t}\n\t\t}\n\t\treturn regress(predIdx);\n\t}\n\t\n\t/**\n\t * Returns the prediction based on prediction index.\n\t * \n\t * @param predIdx\n\t * @return the prediction based on prediction index.\n\t */\n\tpublic double regress(long predIdx) {\n\t\tint idx = Arrays.binarySearch(predIndices, predIdx);\n\t\tif (idx < 0) {\n\t\t\treturn 0;\n\t\t} else {\n\t\t\treturn predValues[idx];\n\t\t}\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/DecisionTableLearner.java",
    "content": "package mltk.predictor.tree;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\nimport mltk.core.Attribute;\nimport mltk.core.Attribute.Type;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.util.ArrayUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.Random;\nimport mltk.util.StatUtils;\nimport mltk.util.tuple.DoublePair;\nimport mltk.util.tuple.IntDoublePair;\nimport mltk.util.tuple.LongDoublePair;\nimport mltk.util.tuple.LongDoublePairComparator;\n\n/**\n * Class for learning decision tables. \n * \n * <p>\n * Reference:<br>\n * Y. Lou and M. Obukhov. BDT: Boosting Decision Tables for High Accuracy and Scoring Efficiency. In <i>Proceedings of the\n * 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD)</i>, Halifax, Nova Scotia, Canada, 2017.\n * </p>\n * \n * This class has a different implementation to better fit the design of this package.\n * \n * @author Yin Lou\n *\n */\npublic class DecisionTableLearner extends RTreeLearner {\n\t\n\t/**\n\t * Enumeration of construction mode.\n\t *\n\t * @author Yin Lou\n\t *\n\t */\n\tpublic enum Mode {\n\n\t\tONE_PASS_GREEDY, MULTI_PASS_CYCLIC, MULTI_PASS_RANDOM;\n\t}\n\t\n\tprotected Mode mode;\n\tprotected int maxDepth;\n\tprotected int numPasses;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic DecisionTableLearner() {\n\t\tmode = Mode.ONE_PASS_GREEDY;\n\t\tmaxDepth = 6;\n\t\tnumPasses = 2;\n\t}\n\t\n\t@Override\n\tpublic void setParameters(String mode) {\n\t\tString[] data = mode.split(\":\");\n\t\tif (data.length != 2) {\n\t\t\tthrow new IllegalArgumentException();\n\t\t}\n\t\tthis.setMaxDepth(Integer.parseInt(data[1]));\n\t\tswitch (data[0]) {\n\t\t\tcase \"g\":\n\t\t\t\tthis.setConstructionMode(Mode.ONE_PASS_GREEDY);\n\t\t\t\tbreak;\n\t\t\tcase \"c\":\n\t\t\t\tthis.setConstructionMode(Mode.MULTI_PASS_CYCLIC);\n\t\t\t\tthis.setNumPasses(2);\n\t\t\t\tbreak;\n\t\t\tcase \"r\":\n\t\t\t\tthis.setConstructionMode(Mode.MULTI_PASS_RANDOM);\n\t\t\t\tthis.setNumPasses(2);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t}\n\t}\n\t\n\t@Override\n\tpublic boolean isRobust() {\n\t\treturn false;\n\t}\n\t\n\t/**\n\t * Returns the construction mode.\n\t *\n\t * @return the construction mode.\n\t */\n\tpublic Mode getConstructionMode() {\n\t\treturn mode;\n\t}\n\t\n\t/**\n\t * Sets the construction mode.\n\t *\n\t * @param mode the construction mode.\n\t */\n\tpublic void setConstructionMode(Mode mode) {\n\t\tthis.mode = mode;\n\t}\n\t\n\t/**\n\t * Returns the maximum depth.\n\t * \n\t * @return the maximum depth.\n\t */\n\tpublic int getMaxDepth() {\n\t\treturn maxDepth;\n\t}\n\n\t/**\n\t * Sets the maximum depth.\n\t *\n\t * @param maxDepth the maximum depth.\n\t */\n\tpublic void setMaxDepth(int maxDepth) {\n\t\tthis.maxDepth = maxDepth;\n\t}\n\t\n\t/**\n\t * Returns the number of passes. This parameter is used in multi-pass cyclic mode.\n\t * \n\t * @return the number of passes.\n\t */\n\tpublic int getNumPasses() {\n\t\treturn numPasses;\n\t}\n\t\n\t/**\n\t * Sets the number of passes.\n\t * \n\t * @param numPasses the number of passes.\n\t */\n\tpublic void setNumPasses(int numPasses) {\n\t\tthis.numPasses = numPasses;\n\t}\n\n\t@Override\n\tpublic DecisionTable build(Instances instances) {\n\t\tDecisionTable ot = null;\n\t\tswitch (mode) {\n\t\t\tcase ONE_PASS_GREEDY:\n\t\t\t\tot = buildOnePassGreedy(instances, maxDepth);\n\t\t\t\tbreak;\n\t\t\tcase MULTI_PASS_CYCLIC:\n\t\t\t\tot = buildMultiPassCyclic(instances, maxDepth, numPasses);\n\t\t\t\tbreak;\n\t\t\tcase MULTI_PASS_RANDOM:\n\t\t\t\tot = buildMultiPassRandom(instances, maxDepth, numPasses);\n\t\t\tdefault:\n\t\t\t\tbreak;\n\t\t}\n\t\treturn ot;\n\t}\n\t\n\t/**\n\t * Builds a standard oblivious regression tree using greedy tree induction.\n\t * \n\t * @param instances the training set.\n\t * @param maxDepth the maximum depth.\n\t * @return an oblivious regression tree.\n\t */\n\tpublic DecisionTable buildOnePassGreedy(Instances instances, int maxDepth) {\n\t\t// stats[0]: totalWeights\n\t\t// stats[1]: sum\n\t\t// stats[2]: weightedMean\n\t\tdouble[] stats = new double[3];\n\t\tMap<Long, Dataset> map = new HashMap<>(instances.size());\n\t\tList<Integer> attList = new ArrayList<>(maxDepth);\n\t\tList<Double> splitList = new ArrayList<>(maxDepth);\n\t\tDataset dataset = null;\n\t\tif (this.cache != null) {\n\t\t\tdataset = Dataset.create(this.cache, instances);\n\t\t} else {\n\t\t\tdataset = Dataset.create(instances);\n\t\t}\n\t\tmap.put(Long.valueOf(0L), dataset);\n\t\t\n\t\tif (maxDepth <= 0) {\n\t\t\tgetStats(dataset.instances, stats);\n\t\t\tfinal double weightedMean = stats[2];\n\t\t\treturn new DecisionTable(\n\t\t\t\t\tnew int[] {},\n\t\t\t\t\tnew double[] {}, \n\t\t\t\t\tnew long[] { 0L },\n\t\t\t\t\tnew double[] { weightedMean });\n\t\t}\n\t\t\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tList<List<Double>> featureValues = new ArrayList<>(attributes.size());\n\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\tAttribute attribute = attributes.get(j);\n\t\t\tList<Double> values = new ArrayList<>();\n\t\t\t\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint numBins = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tfor (int i = 0; i < numBins; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint cardinality = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tfor (int i = 0; i < cardinality; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tSet<Double> set = new HashSet<>();\n\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\tset.add(instance.getValue(attribute));\n\t\t\t\t}\n\t\t\t\tvalues.addAll(set);\n\t\t\t\tCollections.sort(values);\n\t\t\t}\n\t\t\tfeatureValues.add(values);\n\t\t}\n\t\t\n\t\tfor (int d = 0; d < maxDepth; d++) {\n\t\t\tdouble bestGain = Double.NEGATIVE_INFINITY;\n\t\t\tList<IntDoublePair> splitCandidates = new ArrayList<>();\n\t\t\t\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\tList<Double> values = featureValues.get(j);\n\t\t\t\tif (values.size() <= 1) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tAttribute attribute = attributes.get(j);\n\t\t\t\tint attIndex = attribute.getIndex();\n\t\t\t\tString attName = attribute.getName();\n\t\t\t\t\n\t\t\t\tdouble[] gains = new double[values.size() - 1];\n\t\t\t\tfor (Dataset data : map.values()) {\n\t\t\t\t\tgetStats(data.instances, stats);\n\t\t\t\t\tfinal double totalWeights = stats[0];\n\t\t\t\t\tfinal double sum = stats[1];\n\t\t\t\t\t\n\t\t\t\t\tList<IntDoublePair> sortedList = data.sortedLists.get(attName);\n\t\t\t\t\tList<Double> uniqueValues = new ArrayList<>(sortedList.size());\n\t\t\t\t\tList<DoublePair> histogram = new ArrayList<>(sortedList.size());\n\t\t\t\t\tgetHistogram(data.instances, sortedList, uniqueValues, totalWeights, sum, histogram);\n\t\t\t\t\tdouble[] localGains = evalSplits(uniqueValues, histogram, totalWeights, sum);\n\t\t\t\t\tprocessGains(uniqueValues, localGains, values, gains);\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tint idx = StatUtils.indexOfMax(gains);\n\t\t\t\tif (bestGain <= gains[idx]) {\n\t\t\t\t\tdouble split = (values.get(idx) + values.get(idx + 1)) / 2;\n\t\t\t\t\tif (bestGain < gains[idx]) {\n\t\t\t\t\t\tbestGain = gains[idx];\n\t\t\t\t\t\tsplitCandidates.clear();\n\t\t\t\t\t}\n\t\t\t\t\tsplitCandidates.add(new IntDoublePair(attIndex, split));\n\t\t\t\t}\n\t\t\t}\n\t\t\t\n\t\t\tif (splitCandidates.size() == 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\t\n\t\t\tRandom rand = Random.getInstance();\n\t\t\tIntDoublePair split = splitCandidates.get(rand.nextInt(splitCandidates.size()));\n\t\t\tattList.add(split.v1);\n\t\t\tsplitList.add(split.v2);\n\t\t\t\n\t\t\tMap<Long, Dataset> mapNew = new HashMap<>(map.size() * 2);\n\t\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\t\tLong key = entry.getKey();\n\t\t\t\tDataset data = entry.getValue();\n\t\t\t\tDataset left = new Dataset(data.instances);\n\t\t\t\tDataset right = new Dataset(data.instances);\n\t\t\t\tdata.split(split.v1, split.v2, left, right);\n\t\t\t\tif (left.instances.size() > 0) {\n\t\t\t\t\tLong leftKey = (key << 1) | 1L;\n\t\t\t\t\tmapNew.put(leftKey, left);\n\t\t\t\t}\n\t\t\t\tif (right.instances.size() > 0) {\n\t\t\t\t\tLong rightKey = key << 1;\n\t\t\t\t\tmapNew.put(rightKey, right);\n\t\t\t\t}\n\t\t\t}\n\t\t\tmap = mapNew;\n\t\t}\n\t\t\n\t\tint[] attIndices = ArrayUtils.toIntArray(attList);\n\t\tdouble[] splits = ArrayUtils.toDoubleArray(splitList);\n\t\t\n\t\tList<LongDoublePair> list = new ArrayList<>(splits.length);\n\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\tLong key = entry.getKey();\n\t\t\tDataset data = entry.getValue();\n\t\t\tgetStats(data.instances, stats);\n\t\t\tlist.add(new LongDoublePair(key, stats[2]));\n\t\t}\n\t\tCollections.sort(list, new LongDoublePairComparator());\n\t\tlong[] predIndices = new long[list.size()];\n\t\tdouble[] predValues = new double[list.size()];\n\t\tfor (int i = 0; i < predIndices.length; i++) {\n\t\t\tLongDoublePair pair = list.get(i);\n\t\t\tpredIndices[i] = pair.v1;\n\t\t\tpredValues[i] = pair.v2;\n\t\t}\n\t\t\n\t\treturn new DecisionTable(attIndices, splits, predIndices, predValues);\n\t}\n\t\n\t/**\n\t * Builds an oblivious regression tree using multi-pass cyclic backfitting.\n\t * \n\t * @param instances the training set.\n\t * @param maxDepth the maximum depth.\n\t * @param numPasses the number of passes.\n\t * @return an oblivious regression tree.\n\t */\n\tpublic DecisionTable buildMultiPassCyclic(Instances instances, int maxDepth, int numPasses) {\n\t\t// stats[0]: totalWeights\n\t\t// stats[1]: sum\n\t\t// stats[2]: weightedMean\n\t\tdouble[] stats = new double[3];\n\t\tMap<Long, Dataset> map = new HashMap<>(instances.size());\n\t\tint[] attIndices = new int[maxDepth];\n\t\tdouble[] splits = new double[maxDepth];\n\t\tDataset dataset = null;\n\t\tif (this.cache != null) {\n\t\t\tdataset = Dataset.create(this.cache, instances);\n\t\t} else {\n\t\t\tdataset = Dataset.create(instances);\n\t\t}\n\t\tmap.put(Long.valueOf(0L), dataset);\n\t\t\n\t\tif (maxDepth <= 0) {\n\t\t\tgetStats(dataset.instances, stats);\n\t\t\tfinal double weightedMean = stats[2];\n\t\t\treturn new DecisionTable(\n\t\t\t\t\tnew int[] {},\n\t\t\t\t\tnew double[] {}, \n\t\t\t\t\tnew long[] { 0L },\n\t\t\t\t\tnew double[] { weightedMean });\n\t\t}\n\t\t\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tList<List<Double>> featureValues = new ArrayList<>(attributes.size());\n\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\tAttribute attribute = attributes.get(j);\n\t\t\tList<Double> values = new ArrayList<>();\n\t\t\t\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint numBins = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tfor (int i = 0; i < numBins; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint cardinality = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tfor (int i = 0; i < cardinality; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tSet<Double> set = new HashSet<>();\n\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\tset.add(instance.getValue(attribute));\n\t\t\t\t}\n\t\t\t\tvalues.addAll(set);\n\t\t\t\tCollections.sort(values);\n\t\t\t}\n\t\t\tfeatureValues.add(values);\n\t\t}\n\t\t\n\t\tfor (int pass = 0; pass < numPasses; pass++) {\n\t\t\tfor (int d = 0; d < maxDepth; d++) {\n\t\t\t\tdouble bestGain = Double.NEGATIVE_INFINITY;\n\t\t\t\tList<IntDoublePair> splitCandidates = new ArrayList<>();\n\t\t\t\t\n\t\t\t\t// Remove depth d\n\t\t\t\tSet<Long> processedKeys = new HashSet<>();\n\t\t\t\tMap<Long, Dataset> mapNew = new HashMap<>(map.size());\n\t\t\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\t\t\tLong key = entry.getKey();\n\t\t\t\t\tif (processedKeys.contains(key)) {\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\tDataset data = entry.getValue();\n\t\t\t\t\tint s = maxDepth - d - 1;\n\t\t\t\t\tLong otherKey = key ^ (1L << s);\n\t\t\t\t\tif (map.containsKey(otherKey)) {\n\t\t\t\t\t\tlong check = (key >> s) & 1;\n\t\t\t\t\t\tDataset left = null;\n\t\t\t\t\t\tDataset right = null;\n\t\t\t\t\t\tif (check > 0) {\n\t\t\t\t\t\t\tleft = data;\n\t\t\t\t\t\t\tright = map.get(otherKey);\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tleft = map.get(otherKey);\n\t\t\t\t\t\t\tright = data;\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// This key will be updated anyway\n\t\t\t\t\t\tmapNew.put(key, Dataset.merge(left, right));\n\t\t\t\t\t\tprocessedKeys.add(key);\n\t\t\t\t\t\tprocessedKeys.add(otherKey);\n\t\t\t\t\t} else {\n\t\t\t\t\t\tmapNew.put(key, data);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tmap = mapNew;\n\t\t\t\t\n\t\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t\tAttribute attribute = attributes.get(j);\n\t\t\t\t\tint attIndex = attribute.getIndex();\n\t\t\t\t\tString attName = attribute.getName();\n\t\t\t\t\t\n\t\t\t\t\tList<Double> values = featureValues.get(j);\n\t\t\t\t\tif (values.size() <= 1) {\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tdouble[] gains = new double[values.size() - 1];\n\t\t\t\t\tfor (Dataset data : map.values()) {\n\t\t\t\t\t\tgetStats(data.instances, stats);\n\t\t\t\t\t\tfinal double totalWeights = stats[0];\n\t\t\t\t\t\tfinal double sum = stats[1];\n\t\t\t\t\t\t\n\t\t\t\t\t\tList<IntDoublePair> sortedList = data.sortedLists.get(attName);\n\t\t\t\t\t\tList<Double> uniqueValues = new ArrayList<>(sortedList.size());\n\t\t\t\t\t\tList<DoublePair> histogram = new ArrayList<>(sortedList.size());\n\t\t\t\t\t\tgetHistogram(data.instances, sortedList, uniqueValues, totalWeights, sum, histogram);\n\t\t\t\t\t\tdouble[] localGains = evalSplits(uniqueValues, histogram, totalWeights, sum);\n\t\t\t\t\t\tprocessGains(uniqueValues, localGains, values, gains);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tint idx = StatUtils.indexOfMax(gains);\n\t\t\t\t\tif (bestGain <= gains[idx]) {\n\t\t\t\t\t\tdouble split = (values.get(idx) + values.get(idx + 1)) / 2;\n\t\t\t\t\t\tif (bestGain < gains[idx]) {\n\t\t\t\t\t\t\tbestGain = gains[idx];\n\t\t\t\t\t\t\tsplitCandidates.clear();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsplitCandidates.add(new IntDoublePair(attIndex, split));\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tif (splitCandidates.size() == 0) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tRandom rand = Random.getInstance();\n\t\t\t\tIntDoublePair split = splitCandidates.get(rand.nextInt(splitCandidates.size()));\n\t\t\t\tattIndices[d] = split.v1;\n\t\t\t\tsplits[d] = split.v2;\n\t\t\t\t\n\t\t\t\tmapNew = new HashMap<>(map.size() * 2);\n\t\t\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\t\t\tLong key = entry.getKey();\n\t\t\t\t\tDataset data = entry.getValue();\n\t\t\t\t\tDataset left = new Dataset(data.instances);\n\t\t\t\t\tDataset right = new Dataset(data.instances);\n\t\t\t\t\tdata.split(split.v1, split.v2, left, right);\n\t\t\t\t\tint s = maxDepth - d - 1;\n\t\t\t\t\tif (left.instances.size() > 0) {\n\t\t\t\t\t\tLong leftKey = key | (1L << s);\n\t\t\t\t\t\tmapNew.put(leftKey, left);\n\t\t\t\t\t}\n\t\t\t\t\tif (right.instances.size() > 0) {\n\t\t\t\t\t\tLong rightKey = key & ~(1L << s);\n\t\t\t\t\t\tmapNew.put(rightKey, right);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tmap = mapNew;\n\t\t\t}\n\t\t}\n\t\t\n\t\tList<LongDoublePair> list = new ArrayList<>(splits.length);\n\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\tLong key = entry.getKey();\n\t\t\tDataset data = entry.getValue();\n\t\t\tgetStats(data.instances, stats);\n\t\t\tlist.add(new LongDoublePair(key, stats[2]));\n\t\t}\n\t\tCollections.sort(list, new LongDoublePairComparator());\n\t\tlong[] predIndices = new long[list.size()];\n\t\tdouble[] predValues = new double[list.size()];\n\t\tfor (int i = 0; i < predIndices.length; i++) {\n\t\t\tLongDoublePair pair = list.get(i);\n\t\t\tpredIndices[i] = pair.v1;\n\t\t\tpredValues[i] = pair.v2;\n\t\t}\n\t\t\n\t\treturn new DecisionTable(attIndices, splits, predIndices, predValues);\n\t}\n\t\n\t/**\n\t * Builds an oblivious regression tree using multi-pass random backfitting.\n\t * \n\t * @param instances the training set.\n\t * @param maxDepth the maximum depth.\n\t * @param numPasses the number of passes.\n\t * @return an oblivious regression tree.\n\t */\n\tpublic DecisionTable buildMultiPassRandom(Instances instances, int maxDepth, int numPasses) {\n\t\t// stats[0]: totalWeights\n\t\t// stats[1]: sum\n\t\t// stats[2]: weightedMean\n\t\tdouble[] stats = new double[3];\n\t\tMap<Long, Dataset> map = new HashMap<>(instances.size());\n\t\tint[] attIndices = new int[maxDepth];\n\t\tdouble[] splits = new double[maxDepth];\n\t\tDataset dataset = null;\n\t\tif (this.cache != null) {\n\t\t\tdataset = Dataset.create(this.cache, instances);\n\t\t} else {\n\t\t\tdataset = Dataset.create(instances);\n\t\t}\n\t\tmap.put(Long.valueOf(0L), dataset);\n\t\t\n\t\tif (maxDepth <= 0) {\n\t\t\tgetStats(dataset.instances, stats);\n\t\t\tfinal double weightedMean = stats[2];\n\t\t\treturn new DecisionTable(\n\t\t\t\t\tnew int[] {},\n\t\t\t\t\tnew double[] {}, \n\t\t\t\t\tnew long[] { 0L },\n\t\t\t\t\tnew double[] { weightedMean });\n\t\t}\n\t\t\n\t\tList<Attribute> attributes = instances.getAttributes();\n\t\tList<List<Double>> featureValues = new ArrayList<>(attributes.size());\n\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\tAttribute attribute = attributes.get(j);\n\t\t\tList<Double> values = new ArrayList<>();\n\t\t\t\n\t\t\tif (attribute.getType() == Type.BINNED) {\n\t\t\t\tint numBins = ((BinnedAttribute) attribute).getNumBins();\n\t\t\t\tfor (int i = 0; i < numBins; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else if (attribute.getType() == Type.NOMINAL) {\n\t\t\t\tint cardinality = ((NominalAttribute) attribute).getCardinality();\n\t\t\t\tfor (int i = 0; i < cardinality; i++) {\n\t\t\t\t\tvalues.add((double) i);\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tSet<Double> set = new HashSet<>();\n\t\t\t\tfor (Instance instance : instances) {\n\t\t\t\t\tset.add(instance.getValue(attribute));\n\t\t\t\t}\n\t\t\t\tvalues.addAll(set);\n\t\t\t\tCollections.sort(values);\n\t\t\t}\n\t\t\tfeatureValues.add(values);\n\t\t}\n\t\t\n\t\tfor (int iter = 0; iter < numPasses; iter++) {\n\t\t\tfor (int k = 0; k < maxDepth; k++) {\n\t\t\t\tdouble bestGain = Double.NEGATIVE_INFINITY;\n\t\t\t\tList<IntDoublePair> splitCandidates = new ArrayList<>();\n\t\t\t\t\n\t\t\t\tint d = k;\n\t\t\t\tif (iter > 0) {\n\t\t\t\t\td = Random.getInstance().nextInt(maxDepth);\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\t// Remove depth d\n\t\t\t\tSet<Long> processedKeys = new HashSet<>();\n\t\t\t\tMap<Long, Dataset> mapNew = new HashMap<>(map.size());\n\t\t\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\t\t\tLong key = entry.getKey();\n\t\t\t\t\tif (processedKeys.contains(key)) {\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\tDataset data = entry.getValue();\n\t\t\t\t\tint s = maxDepth - d - 1;\n\t\t\t\t\tLong otherKey = key ^ (1L << s);\n\t\t\t\t\tif (map.containsKey(otherKey)) {\n\t\t\t\t\t\tlong check = (key >> s) & 1;\n\t\t\t\t\t\tDataset left = null;\n\t\t\t\t\t\tDataset right = null;\n\t\t\t\t\t\tif (check > 0) {\n\t\t\t\t\t\t\tleft = data;\n\t\t\t\t\t\t\tright = map.get(otherKey);\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tleft = map.get(otherKey);\n\t\t\t\t\t\t\tright = data;\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// This key will be updated anyway\n\t\t\t\t\t\tmapNew.put(key, Dataset.merge(left, right));\n\t\t\t\t\t\tprocessedKeys.add(key);\n\t\t\t\t\t\tprocessedKeys.add(otherKey);\n\t\t\t\t\t} else {\n\t\t\t\t\t\tmapNew.put(key, data);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tmap = mapNew;\n\t\t\t\t\n\t\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\t\tAttribute attribute = attributes.get(j);\n\t\t\t\t\tint attIndex = attribute.getIndex();\n\t\t\t\t\tString attName = attribute.getName();\n\t\t\t\t\t\n\t\t\t\t\tList<Double> values = featureValues.get(j);\n\t\t\t\t\tif (values.size() <= 1) {\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tdouble[] gains = new double[values.size() - 1];\n\t\t\t\t\tfor (Dataset data : map.values()) {\n\t\t\t\t\t\tgetStats(data.instances, stats);\n\t\t\t\t\t\tfinal double totalWeights = stats[0];\n\t\t\t\t\t\tfinal double sum = stats[1];\n\t\t\t\t\t\t\n\t\t\t\t\t\tList<IntDoublePair> sortedList = data.sortedLists.get(attName);\n\t\t\t\t\t\tList<Double> uniqueValues = new ArrayList<>(sortedList.size());\n\t\t\t\t\t\tList<DoublePair> histogram = new ArrayList<>(sortedList.size());\n\t\t\t\t\t\tgetHistogram(data.instances, sortedList, uniqueValues, totalWeights, sum, histogram);\n\t\t\t\t\t\tdouble[] localGains = evalSplits(uniqueValues, histogram, totalWeights, sum);\n\t\t\t\t\t\tprocessGains(uniqueValues, localGains, values, gains);\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tint idx = StatUtils.indexOfMax(gains);\n\t\t\t\t\tif (bestGain <= gains[idx]) {\n\t\t\t\t\t\tdouble split = (values.get(idx) + values.get(idx + 1)) / 2;\n\t\t\t\t\t\tif (bestGain < gains[idx]) {\n\t\t\t\t\t\t\tbestGain = gains[idx];\n\t\t\t\t\t\t\tsplitCandidates.clear();\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsplitCandidates.add(new IntDoublePair(attIndex, split));\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tif (splitCandidates.size() == 0) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\t\n\t\t\t\tRandom rand = Random.getInstance();\n\t\t\t\tIntDoublePair split = splitCandidates.get(rand.nextInt(splitCandidates.size()));\n\t\t\t\tattIndices[d] = split.v1;\n\t\t\t\tsplits[d] = split.v2;\n\t\t\t\t\n\t\t\t\tmapNew = new HashMap<>(map.size() * 2);\n\t\t\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\t\t\tLong key = entry.getKey();\n\t\t\t\t\tDataset data = entry.getValue();\n\t\t\t\t\tDataset left = new Dataset(data.instances);\n\t\t\t\t\tDataset right = new Dataset(data.instances);\n\t\t\t\t\tdata.split(split.v1, split.v2, left, right);\n\t\t\t\t\tint s = maxDepth - d - 1;\n\t\t\t\t\tif (left.instances.size() > 0) {\n\t\t\t\t\t\tLong leftKey = key | (1L << s);\n\t\t\t\t\t\tmapNew.put(leftKey, left);\n\t\t\t\t\t}\n\t\t\t\t\tif (right.instances.size() > 0) {\n\t\t\t\t\t\tLong rightKey = key & ~(1L << s);\n\t\t\t\t\t\tmapNew.put(rightKey, right);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tmap = mapNew;\n\t\t\t}\n\t\t}\n\t\t\n\t\tList<LongDoublePair> list = new ArrayList<>(splits.length);\n\t\tfor (Map.Entry<Long, Dataset> entry : map.entrySet()) {\n\t\t\tLong key = entry.getKey();\n\t\t\tDataset data = entry.getValue();\n\t\t\tgetStats(data.instances, stats);\n\t\t\tlist.add(new LongDoublePair(key, stats[2]));\n\t\t}\n\t\tCollections.sort(list, new LongDoublePairComparator());\n\t\tlong[] predIndices = new long[list.size()];\n\t\tdouble[] predValues = new double[list.size()];\n\t\tfor (int i = 0; i < predIndices.length; i++) {\n\t\t\tLongDoublePair pair = list.get(i);\n\t\t\tpredIndices[i] = pair.v1;\n\t\t\tpredValues[i] = pair.v2;\n\t\t}\n\t\t\n\t\treturn new DecisionTable(attIndices, splits, predIndices, predValues);\n\t}\n\t\n\tprotected void processGains(List<Double> uniqueValues, double[] localGains, List<Double> values, double[] gains) {\n\t\tint i = 0;\n\t\tint j = 0;\n\t\tdouble noSplitGain = localGains[localGains.length - 1];\n\t\tdouble minV = uniqueValues.get(0);\n\t\twhile (j < gains.length) {\n\t\t\tdouble v2 = values.get(j);\n\t\t\tif (v2 < minV) {\n\t\t\t\tgains[j] += noSplitGain;\n\t\t\t\tj++;\n\t\t\t} else {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\tdouble prevGain = localGains[i];\n\t\twhile (i < localGains.length && j < gains.length) {\n\t\t\tdouble v1 = uniqueValues.get(i);\n\t\t\tdouble v2 = values.get(j);\n\t\t\tif (v1 == v2) {\n\t\t\t\tgains[j] += localGains[i];\n\t\t\t\tprevGain = localGains[i];\n\t\t\t\ti++;\n\t\t\t\tj++;\n\t\t\t}\n\t\t\twhile (v1 > v2) {\n\t\t\t\tgains[j] += prevGain;\n\t\t\t\tj++;\n\t\t\t\tv2 = values.get(j);\n\t\t\t}\n\t\t}\n\t\twhile (j < gains.length) {\n\t\t\tgains[j] += noSplitGain;\n\t\t\tj++;\n\t\t}\n\t}\n\t\n\tprotected double[] evalSplits(List<Double> uniqueValues, List<DoublePair> hist, double totalWeights, double sum) {\n\t\tdouble weight1 = hist.get(0).v1;\n\t\tdouble weight2 = totalWeights - weight1;\n\t\tdouble sum1 = hist.get(0).v2;\n\t\tdouble sum2 = sum - sum1;\n\n\t\tdouble[] gains = new double[uniqueValues.size()];\n\t\t\n\t\tgains[0] = OptimUtils.getGain(sum1, weight1) + OptimUtils.getGain(sum2, weight2);\n\t\tfor (int i = 1; i < uniqueValues.size() - 1; i++) {\n\t\t\tfinal double w = hist.get(i).v1;\n\t\t\tfinal double s = hist.get(i).v2;\n\t\t\tweight1 += w;\n\t\t\tweight2 -= w;\n\t\t\tsum1 += s;\n\t\t\tsum2 -= s;\n\t\t\tgains[i] = OptimUtils.getGain(sum1, weight1) + OptimUtils.getGain(sum2, weight2);\n\t\t}\n\t\t// gain for no split\n\t\tgains[uniqueValues.size() - 1] = OptimUtils.getGain(sum, totalWeights);\n\t\t\n\t\treturn gains;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/RTree.java",
    "content": "package mltk.predictor.tree;\n\nimport mltk.predictor.Regressor;\n\n/**\n * Interface for regression trees.\n * \n * @author Yin Lou\n *\n */\npublic interface RTree extends Regressor {\n\n\t/**\n\t * Multiplies this tree with a constant.\n\t * \n\t * @param c the constant.\n\t */\n\tpublic void multiply(double c);\n\t\n\t/**\n\t * Returns a deep copy of this tree.\n\t */\n\tpublic RTree copy();\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/RTreeLearner.java",
    "content": "package mltk.predictor.tree;\n\nimport java.util.Collections;\nimport java.util.List;\n\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.util.tuple.DoublePair;\nimport mltk.util.tuple.IntDoublePair;\n\n/**\n * Abstract class for learning regression trees.\n * \n * @author Yin Lou\n *\n */\npublic abstract class RTreeLearner extends TreeLearner {\n\t\n\t@Override\n\tpublic abstract RTree build(Instances instances);\n\t\n\tprotected void getHistogram(Instances instances, List<IntDoublePair> pairs, List<Double> uniqueValues, double w,\n\t\t\tdouble s, List<DoublePair> histogram) {\n\t\tif (pairs.size() > 0) {\n\t\t\tdouble lastValue = pairs.get(0).v2;\n\t\t\tdouble totalWeight = instances.get(pairs.get(0).v1).getWeight();\n\t\t\tdouble sum = instances.get(pairs.get(0).v1).getTarget() * totalWeight;\n\n\t\t\tfor (int i = 1; i < pairs.size(); i++) {\n\t\t\t\tIntDoublePair pair = pairs.get(i);\n\t\t\t\tdouble value = pair.v2;\n\t\t\t\tdouble weight = instances.get(pairs.get(i).v1).getWeight();\n\t\t\t\tdouble resp = instances.get(pairs.get(i).v1).getTarget();\n\t\t\t\tif (value != lastValue) {\n\t\t\t\t\tuniqueValues.add(lastValue);\n\t\t\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t\t\t\tlastValue = value;\n\t\t\t\t\ttotalWeight = weight;\n\t\t\t\t\tsum = resp * weight;\n\t\t\t\t} else {\n\t\t\t\t\ttotalWeight += weight;\n\t\t\t\t\tsum += resp * weight;\n\t\t\t\t}\n\t\t\t}\n\t\t\tuniqueValues.add(lastValue);\n\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t}\n\n\t\tif (pairs.size() != instances.size()) {\n\t\t\t// Zero entries are present\n\t\t\tdouble sumWeight = 0;\n\t\t\tdouble sumTarget = 0;\n\t\t\tfor (DoublePair pair : histogram) {\n\t\t\t\tsumWeight += pair.v1;\n\t\t\t\tsumTarget += pair.v2;\n\t\t\t}\n\n\t\t\tdouble weightOnZero = w - sumWeight;\n\t\t\tdouble sumOnZero = s - sumTarget;\n\t\t\tint idx = Collections.binarySearch(uniqueValues, ZERO);\n\t\t\tif (idx < 0) {\n\t\t\t\t// This should always happen\n\t\t\t\tuniqueValues.add(-idx - 1, ZERO);\n\t\t\t\thistogram.add(-idx - 1, new DoublePair(weightOnZero, sumOnZero));\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected boolean getStats(Instances instances, double[] stats) {\n\t\tstats[0] = stats[1] = stats[2] = 0;\n\t\tif (instances.size() == 0) {\n\t\t\treturn true;\n\t\t}\n\t\tdouble firstTarget = instances.get(0).getTarget();\n\t\tboolean stdIs0 = true;\n\t\tfor (Instance instance : instances) {\n\t\t\tdouble weight = instance.getWeight();\n\t\t\tdouble target = instance.getTarget();\n\t\t\tstats[0] += weight;\n\t\t\tstats[1] += weight * target;\n\t\t\tif (stdIs0 && target != firstTarget) {\n\t\t\t\tstdIs0 = false;\n\t\t\t}\n\t\t}\n\t\tstats[2] = stats[1] / stats[0];\n\t\tif (Double.isNaN(stats[2])) {\n\t\t\tstats[2] = 0;\n\t\t}\n\t\treturn stdIs0;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/RegressionTree.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\nimport mltk.core.Instance;\n\n/**\n * Class for regression trees.\n * \n * @author Yin Lou\n * \n */\npublic class RegressionTree implements RTree {\n\n\t/**\n\t * The root of a tree.\n\t */\n\tprotected TreeNode root;\n\n\t/**\n\t * Constructs an empty tree.\n\t */\n\tpublic RegressionTree() {\n\t\troot = null;\n\t}\n\n\t/**\n\t * Constructs a regression tree with specified root.\n\t * \n\t * @param root the root.\n\t */\n\tpublic RegressionTree(TreeNode root) {\n\t\tthis.root = root;\n\t}\n\n\t/**\n\t * Returns the root of this regression tree.\n\t * \n\t * @return the root of this regression tree.\n\t */\n\tpublic TreeNode getRoot() {\n\t\treturn root;\n\t}\n\n\t/**\n\t * Sets the root for this regression tree.\n\t * \n\t * @param root the new root.\n\t */\n\tpublic void setRoot(TreeNode root) {\n\t\tthis.root = root;\n\t}\n\n\t/**\n\t * Returns the leaf node.\n\t * \n\t * @param instance the data point.\n\t * @return the leaf node.\n\t */\n\tpublic RegressionTreeLeaf getLeafNode(Instance instance) {\n\t\tTreeNode node = root;\n\t\twhile (!node.isLeaf()) {\n\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\n\t\t\tif (interiorNode.goLeft(instance)) {\n\t\t\t\tnode = interiorNode.getLeftChild();\n\t\t\t} else {\n\t\t\t\tnode = interiorNode.getRightChild();\n\t\t\t}\n\t\t}\n\t\treturn (RegressionTreeLeaf) node;\n\t}\n\n\t/**\n\t * Multiplies this regression tree with a constant.\n\t * \n\t * @param c the constant.\n\t */\n\tpublic void multiply(double c) {\n\t\tmultiply(root, c);\n\t}\n\n\t/**\n\t * Multiplies this subtree with a constant.\n\t * \n\t * @param node the root of the subtree.\n\t * @param c the constant.\n\t */\n\tprotected void multiply(TreeNode node, double c) {\n\t\tif (node.isLeaf()) {\n\t\t\tRegressionTreeLeaf leaf = (RegressionTreeLeaf) node;\n\t\t\tleaf.prediction *= c;\n\t\t} else {\n\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\n\t\t\tmultiply(interiorNode.left, c);\n\t\t\tmultiply(interiorNode.right, c);\n\t\t}\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\treturn getLeafNode(instance).getPrediction();\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tin.readLine();\n\t\tClass<?> clazz = Class.forName(in.readLine());\n\t\troot = (TreeNode) clazz.getDeclaredConstructor().newInstance();\n\t\troot.read(in);\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println();\n\t\troot.write(out);\n\t}\n\n\t@Override\n\tpublic RegressionTree copy() {\n\t\treturn new RegressionTree(root.copy());\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/RegressionTreeLeaf.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\n/**\n * Class for regression tree leaves.\n * \n * @author Yin Lou\n * \n */\npublic class RegressionTreeLeaf extends TreeNode {\n\n\tprotected double prediction;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic RegressionTreeLeaf() {\n\n\t}\n\n\t/**\n\t * Constructs a leaf node with a constant prediction.\n\t * \n\t * @param prediction the prediction for this leaf node.\n\t */\n\tpublic RegressionTreeLeaf(double prediction) {\n\t\tthis.prediction = prediction;\n\t}\n\n\t@Override\n\tpublic boolean isLeaf() {\n\t\treturn true;\n\t}\n\n\t/**\n\t * Sets the prediction for this leaf node.\n\t * \n\t * @param prediction the prediction for this leaf node.\n\t */\n\tpublic void setPrediction(double prediction) {\n\t\tthis.prediction = prediction;\n\t}\n\n\t/**\n\t * Returns the prediction for this leaf node.\n\t * \n\t * @return the prediction for this leaf node.\n\t */\n\tpublic double getPrediction() {\n\t\treturn prediction;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tprediction = Double.parseDouble(in.readLine().split(\": \")[1]);\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.println(this.getClass().getCanonicalName());\n\t\tout.println(\"Prediction: \" + prediction);\n\t}\n\n\t@Override\n\tpublic RegressionTreeLeaf copy() {\n\t\treturn new RegressionTreeLeaf(prediction);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/RegressionTreeLearner.java",
    "content": "package mltk.predictor.tree;\r\n\r\nimport java.util.ArrayList;\r\nimport java.util.HashMap;\r\nimport java.util.List;\r\nimport java.util.Map;\r\nimport java.util.PriorityQueue;\r\n\r\nimport mltk.cmdline.Argument;\r\nimport mltk.cmdline.CmdLineParser;\r\nimport mltk.cmdline.options.LearnerOptions;\r\nimport mltk.core.Attribute;\r\nimport mltk.core.Instances;\r\nimport mltk.core.io.InstancesReader;\r\nimport mltk.predictor.evaluation.Evaluator;\r\nimport mltk.predictor.io.PredictorWriter;\r\nimport mltk.util.Random;\r\nimport mltk.util.Stack;\r\nimport mltk.util.Element;\r\nimport mltk.util.OptimUtils;\r\nimport mltk.util.tuple.DoublePair;\r\nimport mltk.util.tuple.IntDoublePair;\r\n\r\n/**\r\n * Class for learning regression trees.\r\n *\r\n * @author Yin Lou\r\n *\r\n */\r\npublic class RegressionTreeLearner extends RTreeLearner {\r\n\t\r\n\tstatic class Options extends LearnerOptions {\r\n\r\n\t\t@Argument(name = \"-m\", description = \"construction mode:parameter. Construction mode can be alpha limited (a), depth limited (d), number of leaves limited (l) and minimum leaf size limited (s) (default: a:0.001)\")\r\n\t\tString mode = \"a:0.001\";\r\n\r\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\r\n\t\tlong seed = 0L;\r\n\r\n\t}\r\n\t\r\n\t/**\r\n\t * Trains a regression tree.\r\n\t *\r\n\t * <pre>\r\n\t * Usage: mltk.predictor.tree.RegressionTreeLearner\r\n\t * -t\ttrain set path\r\n\t * [-r]\tattribute file path\r\n\t * [-o]\toutput model path\r\n\t * [-V]\tverbose (default: true)\r\n\t * [-m]\tconstruction mode:parameter. Construction mode can be alpha limited (a), depth limited (d), number of leaves limited (l) and minimum leaf size limited (s) (default: a:0.001)\r\n\t * [-s]\tseed of the random number generator (default: 0)\r\n\t * </pre>\r\n\t *\r\n\t * @param args the command line arguments.\r\n\t * @throws Exception\r\n\t */\r\n\tpublic static void main(String[] args) throws Exception {\r\n\t\tOptions opts = new Options();\r\n\t\tCmdLineParser parser = new CmdLineParser(RegressionTreeLearner.class, opts);\r\n\t\tRegressionTreeLearner learner = new RegressionTreeLearner();\r\n\t\ttry {\r\n\t\t\tparser.parse(args);\r\n\t\t\tlearner.setParameters(opts.mode);\r\n\t\t} catch (IllegalArgumentException e) {\r\n\t\t\tparser.printUsage();\r\n\t\t\tSystem.exit(1);\r\n\t\t}\r\n\r\n\t\tRandom.getInstance().setSeed(opts.seed);\r\n\r\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\r\n\t\tlong start = System.currentTimeMillis();\r\n\t\tRegressionTree rt = learner.build(trainSet);\r\n\t\tlong end = System.currentTimeMillis();\r\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0 + \" (s).\");\r\n\t\tSystem.out.println(Evaluator.evalRMSE(rt, trainSet));\r\n\r\n\t\tif (opts.outputModelPath != null) {\r\n\t\t\tPredictorWriter.write(rt, opts.outputModelPath);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Enumeration of construction mode.\r\n\t *\r\n\t * @author Yin Lou\r\n\t *\r\n\t */\r\n\tpublic enum Mode {\r\n\r\n\t\tDEPTH_LIMITED, NUM_LEAVES_LIMITED, ALPHA_LIMITED, MIN_LEAF_SIZE_LIMITED;\r\n\t}\r\n\t\r\n\tprotected int maxDepth;\r\n\tprotected int maxNumLeaves;\r\n\tprotected int minLeafSize;\r\n\tprotected double alpha;\r\n\tprotected Mode mode;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic RegressionTreeLearner() {\r\n\t\talpha = 0.01;\r\n\t\tmode = Mode.ALPHA_LIMITED;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic RegressionTree build(Instances instances) {\r\n\t\tRegressionTree rt = null;\r\n\t\tswitch (mode) {\r\n\t\t\tcase ALPHA_LIMITED:\r\n\t\t\t\trt = buildAlphaLimitedTree(instances, alpha);\r\n\t\t\t\tbreak;\r\n\t\t\tcase NUM_LEAVES_LIMITED:\r\n\t\t\t\trt = buildNumLeafLimitedTree(instances, maxNumLeaves);\r\n\t\t\t\tbreak;\r\n\t\t\tcase DEPTH_LIMITED:\r\n\t\t\t\trt = buildDepthLimitedTree(instances, maxDepth);\r\n\t\t\t\tbreak;\r\n\t\t\tcase MIN_LEAF_SIZE_LIMITED:\r\n\t\t\t\trt = buildMinLeafSizeLimitedTree(instances, minLeafSize);\r\n\t\t\tdefault:\r\n\t\t\t\tbreak;\r\n\t\t}\r\n\t\treturn rt;\r\n\t}\r\n\t\r\n\t@Override\r\n\tpublic void setParameters(String mode) {\r\n\t\tString[] data = mode.split(\":\");\r\n\t\tif (data.length != 2) {\r\n\t\t\tthrow new IllegalArgumentException();\r\n\t\t}\r\n\t\tswitch (data[0]) {\r\n\t\t\tcase \"a\":\r\n\t\t\t\tthis.setConstructionMode(Mode.ALPHA_LIMITED);\r\n\t\t\t\tthis.setAlpha(Double.parseDouble(data[1]));\r\n\t\t\t\tbreak;\r\n\t\t\tcase \"d\":\r\n\t\t\t\tthis.setConstructionMode(Mode.DEPTH_LIMITED);\r\n\t\t\t\tthis.setMaxDepth(Integer.parseInt(data[1]));\r\n\t\t\t\tbreak;\r\n\t\t\tcase \"l\":\r\n\t\t\t\tthis.setConstructionMode(Mode.NUM_LEAVES_LIMITED);\r\n\t\t\t\tthis.setMaxNumLeaves(Integer.parseInt(data[1]));\r\n\t\t\t\tbreak;\r\n\t\t\tcase \"s\":\r\n\t\t\t\tthis.setConstructionMode(Mode.MIN_LEAF_SIZE_LIMITED);\r\n\t\t\t\tthis.setMinLeafSize(Integer.parseInt(data[1]));\r\n\t\t\t\tbreak;\r\n\t\t\tdefault:\r\n\t\t\t\tthrow new IllegalArgumentException();\r\n\t\t}\r\n\t}\r\n\t\r\n\t@Override\r\n\tpublic boolean isRobust() {\r\n\t\treturn false;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the alpha.\r\n\t *\r\n\t * @return the alpha.\r\n\t */\r\n\tpublic double getAlpha() {\r\n\t\treturn alpha;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the construction mode.\r\n\t *\r\n\t * @return the construction mode.\r\n\t */\r\n\tpublic Mode getConstructionMode() {\r\n\t\treturn mode;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the maximum depth.\r\n\t *\r\n\t * @return the maximum depth.\r\n\t */\r\n\tpublic int getMaxDepth() {\r\n\t\treturn maxDepth;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the maximum number of leaves.\r\n\t *\r\n\t * @return the maximum number of leaves.\r\n\t */\r\n\tpublic int getMaxNumLeaves() {\r\n\t\treturn maxNumLeaves;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the minimum leaf size.\r\n\t *\r\n\t * @return the minimum leaf size.\r\n\t */\r\n\tpublic int getMinLeafSize() {\r\n\t\treturn minLeafSize;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the alpha. Alpha is the maximum proportion of the training set in the leaf node.\r\n\t *\r\n\t * @param alpha the alpha.\r\n\t */\r\n\tpublic void setAlpha(double alpha) {\r\n\t\tthis.alpha = alpha;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the construction mode.\r\n\t *\r\n\t * @param mode the construction mode.\r\n\t */\r\n\tpublic void setConstructionMode(Mode mode) {\r\n\t\tthis.mode = mode;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the maximum depth.\r\n\t *\r\n\t * @param maxDepth the maximum depth.\r\n\t */\r\n\tpublic void setMaxDepth(int maxDepth) {\r\n\t\tthis.maxDepth = maxDepth;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the maximum number of leaves.\r\n\t *\r\n\t * @param maxNumLeaves the maximum number of leaves.\r\n\t */\r\n\tpublic void setMaxNumLeaves(int maxNumLeaves) {\r\n\t\tthis.maxNumLeaves = maxNumLeaves;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the minimum leaf size.\r\n\t *\r\n\t * @param minLeafSize the minimum leaf size.\r\n\t */\r\n\tpublic void setMinLeafSize(int minLeafSize) {\r\n\t\tthis.minLeafSize = minLeafSize;\r\n\t}\r\n\r\n\tprotected RegressionTree buildAlphaLimitedTree(Instances instances, double alpha) {\r\n\t\tfinal int limit = (int) (alpha * instances.size());\r\n\t\treturn buildMinLeafSizeLimitedTree(instances, limit);\r\n\t}\r\n\r\n\tprotected RegressionTree buildDepthLimitedTree(Instances instances, int maxDepth) {\r\n\t\tRegressionTree tree = new RegressionTree();\r\n\t\tfinal int limit = 5;\r\n\t\t// stats[0]: totalWeights\r\n\t\t// stats[1]: sum\r\n\t\t// stats[2]: weightedMean\r\n\t\t// stats[3]: splitEval\r\n\t\tdouble[] stats = new double[4];\r\n\t\tif (maxDepth <= 0) {\r\n\t\t\tgetStats(instances, stats);\r\n\t\t\ttree.root = new RegressionTreeLeaf(stats[1]);\r\n\t\t\treturn tree;\r\n\t\t}\r\n\t\tMap<TreeNode, Dataset> datasets = new HashMap<>();\r\n\t\tMap<TreeNode, Integer> depths = new HashMap<>();\r\n\t\tDataset dataset = null;\r\n\t\tif (this.cache != null) {\r\n\t\t\tdataset = Dataset.create(this.cache, instances);\r\n\t\t} else {\r\n\t\t\tdataset = Dataset.create(instances);\r\n\t\t}\r\n\t\ttree.root = createNode(dataset, limit, stats);\r\n\t\tPriorityQueue<Element<TreeNode>> q = new PriorityQueue<>();\r\n\t\tq.add(new Element<TreeNode>(tree.root, stats[2]));\r\n\t\tdatasets.put(tree.root, dataset);\r\n\t\tdepths.put(tree.root, 0);\r\n\r\n\t\twhile (!q.isEmpty()) {\r\n\t\t\tElement<TreeNode> elemt = q.remove();\r\n\t\t\tTreeNode node = elemt.element;\r\n\t\t\tDataset data = datasets.get(node);\r\n\t\t\tint depth = depths.get(node);\r\n\t\t\tif (!node.isLeaf()) {\r\n\t\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\r\n\t\t\t\tDataset left = new Dataset(data.instances);\r\n\t\t\t\tDataset right = new Dataset(data.instances);\r\n\t\t\t\tsplit(data, interiorNode, left, right);\r\n\r\n\t\t\t\tif (depth >= maxDepth - 1) {\r\n\t\t\t\t\tgetStats(left.instances, stats);\r\n\t\t\t\t\tinteriorNode.left = new RegressionTreeLeaf(stats[2]);\r\n\t\t\t\t\tgetStats(right.instances, stats);\r\n\t\t\t\t\tinteriorNode.right = new RegressionTreeLeaf(stats[2]);\r\n\t\t\t\t} else {\r\n\t\t\t\t\tinteriorNode.left = createNode(left, limit, stats);\r\n\t\t\t\t\tif (!interiorNode.left.isLeaf()) {\r\n\t\t\t\t\t\tq.add(new Element<TreeNode>(interiorNode.left, stats[3]));\r\n\t\t\t\t\t\tdatasets.put(interiorNode.left, left);\r\n\t\t\t\t\t\tdepths.put(interiorNode.left, depth + 1);\r\n\t\t\t\t\t}\r\n\t\t\t\t\tinteriorNode.right = createNode(right, limit, stats);\r\n\t\t\t\t\tif (!interiorNode.right.isLeaf()) {\r\n\t\t\t\t\t\tq.add(new Element<TreeNode>(interiorNode.right, stats[3]));\r\n\t\t\t\t\t\tdatasets.put(interiorNode.right, right);\r\n\t\t\t\t\t\tdepths.put(interiorNode.right, depth + 1);\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\treturn tree;\r\n\t}\r\n\r\n\tprotected RegressionTree buildMinLeafSizeLimitedTree(Instances instances, int limit) {\r\n\t\tRegressionTree tree = new RegressionTree();\r\n\t\t// stats[0]: totalWeights\r\n\t\t// stats[1]: sum\r\n\t\t// stats[2]: weightedMean\r\n\t\t// stats[3]: splitEval\r\n\t\tdouble[] stats = new double[4];\r\n\t\tDataset dataset = null;\r\n\t\tif (this.cache != null) {\r\n\t\t\tdataset = Dataset.create(this.cache, instances);\r\n\t\t} else {\r\n\t\t\tdataset = Dataset.create(instances);\r\n\t\t}\r\n\t\tStack<TreeNode> nodes = new Stack<>();\r\n\t\tStack<Dataset> datasets = new Stack<>();\r\n\t\ttree.root = createNode(dataset, limit, stats);\r\n\t\tnodes.push(tree.root);\r\n\t\tdatasets.push(dataset);\r\n\t\twhile (!nodes.isEmpty()) {\r\n\t\t\tTreeNode node = nodes.pop();\r\n\t\t\tDataset data = datasets.pop();\r\n\t\t\tif (!node.isLeaf()) {\r\n\t\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\r\n\t\t\t\tDataset left = new Dataset(data.instances);\r\n\t\t\t\tDataset right = new Dataset(data.instances);\r\n\t\t\t\tsplit(data, interiorNode, left, right);\r\n\t\t\t\tinteriorNode.left = createNode(left, limit, stats);\r\n\t\t\t\tinteriorNode.right = createNode(right, limit, stats);\r\n\t\t\t\tnodes.push(interiorNode.left);\r\n\t\t\t\tdatasets.push(left);\r\n\t\t\t\tnodes.push(interiorNode.right);\r\n\t\t\t\tdatasets.push(right);\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn tree;\r\n\t}\r\n\r\n\tprotected RegressionTree buildNumLeafLimitedTree(Instances instances, int maxNumLeaves) {\r\n\t\tRegressionTree tree = new RegressionTree();\r\n\t\tfinal int limit = 5;\r\n\t\t// stats[0]: totalWeights\r\n\t\t// stats[1]: sum\r\n\t\t// stats[2]: weightedMean\r\n\t\t// stats[3]: splitEval\r\n\t\tdouble[] stats = new double[4];\r\n\t\tMap<TreeNode, Double> nodePred = new HashMap<>();\r\n\t\tMap<TreeNode, Dataset> datasets = new HashMap<>();\r\n\t\tDataset dataset = null;\r\n\t\tif (this.cache != null) {\r\n\t\t\tdataset = Dataset.create(this.cache, instances);\r\n\t\t} else {\r\n\t\t\tdataset = Dataset.create(instances);\r\n\t\t}\r\n\t\tPriorityQueue<Element<TreeNode>> q = new PriorityQueue<>();\r\n\t\ttree.root = createNode(dataset, limit, stats);\r\n\t\tq.add(new Element<TreeNode>(tree.root, stats[2]));\r\n\t\tdatasets.put(tree.root, dataset);\r\n\t\tnodePred.put(tree.root, stats[1]);\r\n\r\n\t\tint numLeaves = 0;\r\n\t\twhile (!q.isEmpty()) {\r\n\t\t\tElement<TreeNode> elemt = q.remove();\r\n\t\t\tTreeNode node = elemt.element;\r\n\t\t\tDataset data = datasets.get(node);\r\n\t\t\tif (!node.isLeaf()) {\r\n\t\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\r\n\t\t\t\tDataset left = new Dataset(data.instances);\r\n\t\t\t\tDataset right = new Dataset(data.instances);\r\n\t\t\t\tsplit(data, interiorNode, left, right);\r\n\r\n\t\t\t\tinteriorNode.left = createNode(left, limit, stats);\r\n\t\t\t\tif (!interiorNode.left.isLeaf()) {\r\n\t\t\t\t\tnodePred.put(interiorNode.left, stats[2]);\r\n\t\t\t\t\tq.add(new Element<TreeNode>(interiorNode.left, stats[3]));\r\n\t\t\t\t\tdatasets.put(interiorNode.left, left);\r\n\t\t\t\t} else {\r\n\t\t\t\t\tnumLeaves++;\r\n\t\t\t\t}\r\n\t\t\t\tinteriorNode.right = createNode(right, limit, stats);\r\n\t\t\t\tif (!interiorNode.right.isLeaf()) {\r\n\t\t\t\t\tnodePred.put(interiorNode.right, stats[2]);\r\n\t\t\t\t\tq.add(new Element<TreeNode>(interiorNode.right, stats[3]));\r\n\t\t\t\t\tdatasets.put(interiorNode.right, right);\r\n\t\t\t\t} else {\r\n\t\t\t\t\tnumLeaves++;\r\n\t\t\t\t}\r\n\r\n\t\t\t\tif (numLeaves + q.size() >= maxNumLeaves) {\r\n\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\t// Convert interior nodes to leaves\r\n\t\tMap<TreeNode, TreeNode> parent = new HashMap<>();\r\n\t\ttraverse(tree.root, parent);\r\n\t\twhile (!q.isEmpty()) {\r\n\t\t\tElement<TreeNode> elemt = q.remove();\r\n\t\t\tTreeNode node = elemt.element;\r\n\r\n\t\t\tdouble prediction = nodePred.get(node);\r\n\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) parent.get(node);\r\n\t\t\tif (interiorNode.left == node) {\r\n\t\t\t\tinteriorNode.left = new RegressionTreeLeaf(prediction);\r\n\t\t\t} else {\r\n\t\t\t\tinteriorNode.right = new RegressionTreeLeaf(prediction);\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\treturn tree;\r\n\t}\r\n\r\n\tprotected TreeNode createNode(Dataset dataset, int limit, double[] stats) {\r\n\t\tboolean stdIs0 = getStats(dataset.instances, stats);\r\n\t\tfinal double totalWeights = stats[0];\r\n\t\tfinal double sum = stats[1];\r\n\t\tfinal double weightedMean = stats[2];\r\n\r\n\t\t// 1. Check basic leaf conditions\r\n\t\tif (dataset.instances.size() < limit || stdIs0) {\r\n\t\t\tTreeNode node = new RegressionTreeLeaf(weightedMean);\r\n\t\t\treturn node;\r\n\t\t}\r\n\r\n\t\t// 2. Find best split\r\n\t\tdouble bestEval = Double.POSITIVE_INFINITY;\r\n\t\tList<IntDoublePair> splits = new ArrayList<>();\r\n\t\tList<Attribute> attributes = dataset.instances.getAttributes();\r\n\t\tfor (int j = 0; j < attributes.size(); j++) {\r\n\t\t\tint attIndex = attributes.get(j).getIndex();\r\n\t\t\tString attName = attributes.get(j).getName();\r\n\t\t\tList<IntDoublePair> sortedList = dataset.sortedLists.get(attName);\r\n\t\t\tList<Double> uniqueValues = new ArrayList<>(sortedList.size());\r\n\t\t\tList<DoublePair> histogram = new ArrayList<>(sortedList.size());\r\n\t\t\tgetHistogram(dataset.instances, sortedList, uniqueValues, totalWeights, sum, histogram);\r\n\r\n\t\t\tif (uniqueValues.size() > 1) {\r\n\t\t\t\tDoublePair split = split(uniqueValues, histogram, totalWeights, sum);\r\n\t\t\t\tif (split.v2 <= bestEval) {\r\n\t\t\t\t\tIntDoublePair splitPoint = new IntDoublePair(attIndex, split.v1);\r\n\t\t\t\t\tif (split.v2 < bestEval) {\r\n\t\t\t\t\t\tsplits.clear();\r\n\t\t\t\t\t\tbestEval = split.v2;\r\n\t\t\t\t\t}\r\n\t\t\t\t\tsplits.add(splitPoint);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\tif (bestEval < Double.POSITIVE_INFINITY) {\r\n\t\t\tRandom rand = Random.getInstance();\r\n\t\t\tIntDoublePair splitPoint = splits.get(rand.nextInt(splits.size()));\r\n\t\t\tint attIndex = splitPoint.v1;\r\n\t\t\tTreeNode node = new TreeInteriorNode(attIndex, splitPoint.v2);\r\n\t\t\tstats[3] = bestEval + totalWeights * weightedMean * weightedMean;\r\n\t\t\treturn node;\r\n\t\t} else {\r\n\t\t\tTreeNode node = new RegressionTreeLeaf(weightedMean);\r\n\t\t\treturn node;\r\n\t\t}\r\n\t}\r\n\r\n\tprotected void split(Dataset data, TreeInteriorNode node, Dataset left, Dataset right) {\r\n\t\tdata.split(node.getSplitAttributeIndex(), node.getSplitPoint(), left, right);\r\n\t}\r\n\r\n\tprotected DoublePair split(List<Double> uniqueValues, List<DoublePair> hist, double totalWeights, double sum) {\r\n\t\tdouble weight1 = hist.get(0).v1;\r\n\t\tdouble weight2 = totalWeights - weight1;\r\n\t\tdouble sum1 = hist.get(0).v2;\r\n\t\tdouble sum2 = sum - sum1;\r\n\r\n\t\tdouble bestEval = -(OptimUtils.getGain(sum1, weight1) + OptimUtils.getGain(sum2, weight2));\r\n\t\tList<Double> splits = new ArrayList<>();\r\n\t\tsplits.add((uniqueValues.get(0) + uniqueValues.get(0 + 1)) / 2);\r\n\t\tfor (int i = 1; i < uniqueValues.size() - 1; i++) {\r\n\t\t\tfinal double w = hist.get(i).v1;\r\n\t\t\tfinal double s = hist.get(i).v2;\r\n\t\t\tweight1 += w;\r\n\t\t\tweight2 -= w;\r\n\t\t\tsum1 += s;\r\n\t\t\tsum2 -= s;\r\n\t\t\tdouble eval1 = OptimUtils.getGain(sum1, weight1);\r\n\t\t\tdouble eval2 = OptimUtils.getGain(sum2, weight2);\r\n\t\t\tdouble eval = -(eval1 + eval2);\r\n\t\t\tif (eval <= bestEval) {\r\n\t\t\t\tdouble split = (uniqueValues.get(i) + uniqueValues.get(i + 1)) / 2;\r\n\t\t\t\tif (eval < bestEval) {\r\n\t\t\t\t\tbestEval = eval;\r\n\t\t\t\t\tsplits.clear();\r\n\t\t\t\t}\r\n\t\t\t\tsplits.add(split);\r\n\t\t\t}\r\n\t\t}\r\n\t\tRandom rand = Random.getInstance();\r\n\t\tdouble split = splits.get(rand.nextInt(splits.size()));\r\n\t\treturn new DoublePair(split, bestEval);\r\n\t}\r\n\r\n\tprotected void traverse(TreeNode node, Map<TreeNode, TreeNode> parent) {\r\n\t\tif (!node.isLeaf()) {\r\n\t\t\tTreeInteriorNode interiorNode = (TreeInteriorNode) node;\r\n\t\t\tif (interiorNode.left != null) {\r\n\t\t\t\tparent.put(interiorNode.left, node);\r\n\t\t\t\ttraverse(interiorNode.left, parent);\r\n\t\t\t}\r\n\t\t\tif (interiorNode.right != null) {\r\n\t\t\t\tparent.put(interiorNode.right, node);\r\n\t\t\t\ttraverse(interiorNode.right, parent);\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/TreeInteriorNode.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\nimport mltk.core.Instance;\n\n/**\n * Class for tree interior nodes.\n * \n * @author Yin Lou\n * \n */\npublic class TreeInteriorNode extends TreeNode {\n\n\tprotected TreeNode left;\n\tprotected TreeNode right;\n\tprotected int attIndex;\n\tprotected double splitPoint;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic TreeInteriorNode() {\n\n\t}\n\n\t/**\n\t * Constructs an interior node with attribute index and split point.\n\t * \n\t * @param attIndex the attribute index.\n\t * @param splitPoint the split point.\n\t */\n\tpublic TreeInteriorNode(int attIndex, double splitPoint) {\n\t\tthis.attIndex = attIndex;\n\t\tthis.splitPoint = splitPoint;\n\t}\n\n\t/**\n\t * Returns the left child.\n\t * \n\t * @return the left child.\n\t */\n\tpublic TreeNode getLeftChild() {\n\t\treturn left;\n\t}\n\n\t/**\n\t * Returns the right child.\n\t * \n\t * @return the right child.\n\t */\n\tpublic TreeNode getRightChild() {\n\t\treturn right;\n\t}\n\n\t/**\n\t * Returns the split attribute index.\n\t * \n\t * @return the split attribute index.\n\t */\n\tpublic int getSplitAttributeIndex() {\n\t\treturn attIndex;\n\t}\n\n\t/**\n\t * Returns the split point.\n\t * \n\t * @return the split point.\n\t */\n\tpublic double getSplitPoint() {\n\t\treturn splitPoint;\n\t}\n\n\t@Override\n\tpublic boolean isLeaf() {\n\t\treturn false;\n\t}\n\n\t/**\n\t * Returns {@code true} if going to left child.\n\t * \n\t * @param instance the instance.\n\t * @return {@code true} if going to left child.\n\t */\n\tpublic boolean goLeft(Instance instance) {\n\t\tdouble value = instance.getValue(attIndex);\n\t\treturn value <= splitPoint;\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tattIndex = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tsplitPoint = Double.parseDouble(in.readLine().split(\": \")[1]);\n\t\tin.readLine();\n\n\t\tClass<?> clazzLeft = Class.forName(in.readLine());\n\t\tleft = (TreeNode) clazzLeft.getDeclaredConstructor().newInstance();\n\t\tleft.read(in);\n\n\t\tin.readLine();\n\n\t\tClass<?> clazzRight = Class.forName(in.readLine());\n\t\tright = (TreeNode) clazzRight.getDeclaredConstructor().newInstance();\n\t\tright.read(in);\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.println(this.getClass().getCanonicalName());\n\t\tout.println(\"AttIndex: \" + attIndex);\n\t\tout.println(\"SplintPoint: \" + splitPoint);\n\t\tout.println();\n\t\tleft.write(out);\n\t\tout.println();\n\t\tright.write(out);\n\t}\n\n\t@Override\n\tpublic TreeNode copy() {\n\t\tTreeInteriorNode copy = new TreeInteriorNode(attIndex, splitPoint);\n\t\tcopy.left = this.left.copy();\n\t\tcopy.right = this.right.copy();\n\t\treturn copy;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/TreeLearner.java",
    "content": "package mltk.predictor.tree;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.SparseVector;\nimport mltk.predictor.Learner;\nimport mltk.util.tuple.IntDoublePair;\nimport mltk.util.tuple.IntDoublePairComparator;\n\n/**\n * Abstract class for learning trees.\n * \n * @author Yin Lou\n *\n */\npublic abstract class TreeLearner extends Learner {\n\t\n\tprotected static final Double ZERO = Double.valueOf(0.0);\n\tprotected static final IntDoublePairComparator COMP = new IntDoublePairComparator(false);\n\t\n\tprotected Dataset cache;\n\t\n\t/**\n\t * Returns {@code true} if this tree learner can be used in {@link mltk.predictor.tree.ensemble.brt.LogitBoostLearner}.\n\t * \n\t * @return {@code true} if this tree learner can be used in {@link mltk.predictor.tree.ensemble.brt.LogitBoostLearner}.\n\t */\n\tpublic abstract boolean isRobust();\n\t\n\t/**\n\t * Caches the auxiliary data structures. This method is used in ensemble method\n\t * so that same data structures can be shared across iterations.\n\t * \n\t * @param instances the instances.\n\t */\n\tpublic void cache(Instances instances) {\n\t\tcache = Dataset.create(instances);\n\t}\n\t\n\t/**\n\t * Evicts the cached data structures.\n\t */\n\tpublic void evictCache() {\n\t\tcache = null;\n\t}\n\t\n\t/**\n\t * Sets the parameters for this tree learner.\n\t * \n\t * @param mode the parameters.\n\t */\n\tpublic abstract void setParameters(String mode);\n\n\tprotected static class Dataset {\n\t\t\n\t\tstatic Dataset create(Instances instances) {\n\t\t\tDataset dataset = new Dataset(instances);\n\t\t\tList<Attribute> attributes = instances.getAttributes();\n\t\t\t// From attIndex to attName\n\t\t\tMap<Integer, String> attMap = new HashMap<>();\n\t\t\tfor (int j = 0; j < attributes.size(); j++) {\n\t\t\t\tAttribute attribute = attributes.get(j);\n\t\t\t\tattMap.put(attribute.getIndex(), attribute.getName());\n\t\t\t}\n\t\t\tfor (Attribute attribute : attributes) {\n\t\t\t\tdataset.sortedLists.put(attribute.getName(), new ArrayList<IntDoublePair>());\n\t\t\t}\n\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\tInstance instance = instances.get(i);\n\t\t\t\tdataset.instances.add(instance.clone());\n\t\t\t\tif (instance.isSparse()) {\n\t\t\t\t\tSparseVector sv = (SparseVector) instance.getVector();\n\t\t\t\t\tint[] indices = sv.getIndices();\n\t\t\t\t\tdouble[] values = sv.getValues();\n\t\t\t\t\tfor (int k = 0; k < indices.length; k++) {\n\t\t\t\t\t\tif (attMap.containsKey(indices[k])) {\n\t\t\t\t\t\t\tString attName = attMap.get(indices[k]);\n\t\t\t\t\t\t\tdataset.sortedLists.get(attName).add(new IntDoublePair(i, values[k]));\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tdouble[] values = instance.getValues();\n\t\t\t\t\tfor (int j = 0; j < values.length; j++) {\n\t\t\t\t\t\tif (attMap.containsKey(j) && values[j] != 0.0) {\n\t\t\t\t\t\t\tString attName = attMap.get(j);\n\t\t\t\t\t\t\tdataset.sortedLists.get(attName).add(new IntDoublePair(i, values[j]));\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor (List<IntDoublePair> sortedList : dataset.sortedLists.values()) {\n\t\t\t\tCollections.sort(sortedList, COMP);\n\t\t\t}\n\t\t\treturn dataset;\n\t\t}\n\t\t\n\t\tstatic Dataset create(Dataset dataset, Instances instances) {\n\t\t\tDataset copy = new Dataset();\n\t\t\tcopy.instances = instances;\n\t\t\tcopy.sortedLists = new HashMap<>(instances.dimension());\n\t\t\tList<Attribute> attributes = instances.getAttributes();\n\t\t\tfor (Attribute attribute : attributes) {\n\t\t\t\tString attName = attribute.getName();\n\t\t\t\tList<IntDoublePair> sortedList = dataset.sortedLists.get(attName);\n\t\t\t\tif (sortedList == null) {\n\t\t\t\t\t// This should not happen very often\n\t\t\t\t\tsortedList = new ArrayList<>();\n\t\t\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\t\t\tInstance instance = instances.get(i);\n\t\t\t\t\t\tdouble v = instance.getValue(attribute);\n\t\t\t\t\t\tif (v != 0.0) {\n\t\t\t\t\t\t\tsortedList.add(new IntDoublePair(i, v));\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tCollections.sort(sortedList, COMP);\n\t\t\t\t\tdataset.sortedLists.put(attName, sortedList);\n\t\t\t\t}\n\t\t\t\tList<IntDoublePair> copySortedList = new ArrayList<>(sortedList.size());\n\t\t\t\tfor (IntDoublePair pair : sortedList) {\n\t\t\t\t\tcopySortedList.add(new IntDoublePair(pair.v1, pair.v2));\n\t\t\t\t}\n\t\t\t\tcopy.sortedLists.put(attName, copySortedList);\n\t\t\t}\n\t\t\treturn copy;\n\t\t}\n\n\t\tpublic Instances instances;\n\t\tpublic Map<String, List<IntDoublePair>> sortedLists;\n\t\t\n\t\tDataset() {\n\t\t\t\n\t\t}\n\n\t\tDataset(Instances instances) {\n\t\t\tthis.instances = new Instances(instances.getAttributes(), instances.getTargetAttribute());\n\t\t\tsortedLists = new HashMap<>(instances.dimension());\n\t\t}\n\t\t\n\t\tstatic Dataset merge(Dataset left, Dataset right) {\n\t\t\tDataset data = new Dataset(left.instances);\n\t\t\tint lSize = left.instances.size();\n\t\t\tfor (Instance instance : left.instances) {\n\t\t\t\tdata.instances.add(instance);\n\t\t\t}\n\t\t\tfor (Instance instance : right.instances) {\n\t\t\t\tdata.instances.add(instance);\n\t\t\t}\n\t\t\tfor (String attName : left.sortedLists.keySet()) {\n\t\t\t\tList<IntDoublePair> lSortedList = left.sortedLists.get(attName);\n\t\t\t\tList<IntDoublePair> rSortedList = right.sortedLists.get(attName);\n\t\t\t\tList<IntDoublePair> sortedList = new ArrayList<>(data.instances.size());\n\t\t\t\tint i = 0;\n\t\t\t\tint j = 0;\n\t\t\t\twhile (i < lSortedList.size() && j < rSortedList.size()) {\n\t\t\t\t\tIntDoublePair l = lSortedList.get(i);\n\t\t\t\t\tIntDoublePair r = rSortedList.get(j);\n\t\t\t\t\tif (l.v2 < r.v2) {\n\t\t\t\t\t\tsortedList.add(l);\n\t\t\t\t\t\ti++;\n\t\t\t\t\t} else if (l.v2 > r.v2) {\n\t\t\t\t\t\tr.v1 += + lSize;\n\t\t\t\t\t\tsortedList.add(r);\n\t\t\t\t\t\tj++;\n\t\t\t\t\t} else {\n\t\t\t\t\t\tsortedList.add(l);\n\t\t\t\t\t\tr.v1 += lSize;\n\t\t\t\t\t\tsortedList.add(r);\n\t\t\t\t\t\ti++;\n\t\t\t\t\t\tj++;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twhile (i < lSortedList.size()) {\n\t\t\t\t\tIntDoublePair l = lSortedList.get(i);\n\t\t\t\t\tsortedList.add(l);\n\t\t\t\t\ti++;\n\t\t\t\t}\n\t\t\t\twhile (j < rSortedList.size()) {\n\t\t\t\t\tIntDoublePair r = rSortedList.get(j);\n\t\t\t\t\tr.v1 += lSize;\n\t\t\t\t\tsortedList.add(r);\n\t\t\t\t\tj++;\n\t\t\t\t}\n\t\t\t\tdata.sortedLists.put(attName, sortedList);\n\t\t\t}\n\t\t\t\n\t\t\treturn data;\n\t\t}\n\t\t\n\t\tvoid split(int attIndex, double split, Dataset left, Dataset right) {\n\t\t\tint[] leftHash = new int[instances.size()];\n\t\t\tint[] rightHash = new int[instances.size()];\n\t\t\tArrays.fill(leftHash, -1);\n\t\t\tArrays.fill(rightHash, -1);\n\t\t\tfor (int i = 0; i < instances.size(); i++) {\n\t\t\t\tInstance instance = instances.get(i);\n\t\t\t\tif (instance.getValue(attIndex) <= split) {\n\t\t\t\t\tleft.instances.add(instance);\n\t\t\t\t\tleftHash[i] = left.instances.size() - 1;\n\t\t\t\t} else {\n\t\t\t\t\tright.instances.add(instance);\n\t\t\t\t\trightHash[i] = right.instances.size() - 1;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor (String attName : sortedLists.keySet()) {\n\t\t\t\tleft.sortedLists.put(attName, new ArrayList<IntDoublePair>(left.instances.size()));\n\t\t\t\tright.sortedLists.put(attName, new ArrayList<IntDoublePair>(right.instances.size()));\n\n\t\t\t\tList<IntDoublePair> sortedList = sortedLists.get(attName);\n\t\t\t\tfor (IntDoublePair pair : sortedList) {\n\t\t\t\t\tint leftIdx = leftHash[pair.v1];\n\t\t\t\t\tint rightIdx = rightHash[pair.v1];\n\t\t\t\t\tif (leftIdx != -1) {\n\t\t\t\t\t\tpair.v1 = leftIdx;\n\t\t\t\t\t\tleft.sortedLists.get(attName).add(pair);\n\t\t\t\t\t}\n\t\t\t\t\tif (rightIdx != -1) {\n\t\t\t\t\t\tpair.v1 = rightIdx;\n\t\t\t\t\t\tright.sortedLists.get(attName).add(pair);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/TreeNode.java",
    "content": "package mltk.predictor.tree;\n\nimport mltk.core.Copyable;\nimport mltk.core.Writable;\n\n/**\n * Abstract class for tree nodes.\n * \n * @author Yin Lou\n * \n */\npublic abstract class TreeNode implements Writable, Copyable<TreeNode> {\n\n\t/**\n\t * Returns {@code true} if the node is a leaf.\n\t * \n\t * @return {@code true} if the node is a leaf.\n\t */\n\tpublic abstract boolean isLeaf();\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/BaggedRTrees.java",
    "content": "package mltk.predictor.tree.ensemble;\r\n\r\nimport java.util.ArrayList;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.tree.RTree;\r\n\r\n/**\r\n * Class for bagged regression trees.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BaggedRTrees extends RTreeList {\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic BaggedRTrees() {\r\n\t\tsuper();\r\n\t}\r\n\r\n\t/**\r\n\t * Constructs a regression tree list of length n. By default each tree is null.\r\n\t * \r\n\t * @param n the length.\r\n\t */\r\n\tpublic BaggedRTrees(int n) {\r\n\t\ttrees = new ArrayList<>(n);\r\n\t\tfor (int i = 0; i < n; i++) {\r\n\t\t\ttrees.add(null);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Regresses an instance.\r\n\t * \r\n\t * @param instance the instance.\r\n\t * @return the regressed\r\n\t */\r\n\tpublic double regress(Instance instance) {\r\n\t\tdouble pred = 0;\r\n\t\tfor (RTree rt : trees) {\r\n\t\t\tpred += rt.regress(instance);\r\n\t\t}\r\n\t\treturn pred / trees.size();\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/BoostedDTables.java",
    "content": "package mltk.predictor.tree.ensemble;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.core.Copyable;\nimport mltk.core.Instance;\nimport mltk.core.SparseVector;\nimport mltk.predictor.tree.DecisionTable;\nimport mltk.predictor.tree.RTree;\n\n/**\n * Class for boosted decision tables.\n * \n * @author Yin Lou\n *\n */\npublic class BoostedDTables implements Copyable<BoostedDTables> {\n\t\n\tstatic class IndexElement implements Comparable<IndexElement> {\n\t\t\n\t\tdouble cut;\n\t\tint tid;\n\t\tint pos;\n\t\t\n\t\tpublic IndexElement(double cut, int tid, int pos) {\n\t\t\tthis.cut = cut;\n\t\t\tthis.tid = tid;\n\t\t\tthis.pos = pos;\n\t\t}\n\n\t\t@Override\n\t\tpublic int compareTo(IndexElement o) {\n\t\t\treturn Double.compare(o.cut, this.cut);\n\t\t}\n\t\t\n\t\tpublic IndexElement copy() {\n\t\t\treturn new IndexElement(cut, tid, pos);\n\t\t}\n\t\t\n\t}\n\t\n\tstatic class Index {\n\t\t\n\t\tIndexElement[] elements;\n\t\t\n\t\tpublic Index(IndexElement[] elements) {\n\t\t\tthis.elements = elements;\n\t\t}\n\t\t\n\t\tvoid setPredIdx(long[] predIndices, double v) {\n\t\t\tfor (IndexElement element : elements) {\n\t\t\t\tif (v <= element.cut) {\n\t\t\t\t\tlong t = 1 << element.pos;\n\t\t\t\t\tint tid = element.tid;\n\t\t\t\t\tpredIndices[tid] |= t;\n\t\t\t\t} else {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t\n\t\tpublic Index copy() {\n\t\t\tIndexElement[] elementsCopy = new IndexElement[elements.length];\n\t\t\tfor (int i = 0; i < elementsCopy.length; i++) {\n\t\t\t\telementsCopy[i] = elements[i].copy();\n\t\t\t}\n\t\t\treturn new Index(elementsCopy);\n\t\t}\n\t\t\n\t}\n\t\n\tprotected static final IndexElement[] EMPTY_INDEX = new IndexElement[0];\n\n\tprotected List<DecisionTable> dtList;\n\tprotected Index[] indexes;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic BoostedDTables() {\n\t\tdtList = new ArrayList<>();\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param trees\n\t */\n\tpublic BoostedDTables(BoostedRTrees trees) {\n\t\tdtList = new ArrayList<>(trees.size());\n\t\tfor (RTree tree : trees) {\n\t\t\tdtList.add((DecisionTable) tree);\n\t\t}\n\t\tbuildIndex();\n\t}\n\t\n\t/**\n\t * Builds the index for fast scoring.\n\t */\n\tpublic void buildIndex() {\n\t\tMap<Integer, List<IndexElement>> map = new HashMap<>();\n\t\tint p = -1;\n\t\tfor (int i = 0; i < dtList.size(); i++) {\n\t\t\tDecisionTable dt = dtList.get(i);\n\t\t\tint[] attIndices = dt.getAttributeIndices();\n\t\t\tdouble[] cuts = dt.getSplits();\n\t\t\tfor (int k = 0; k < attIndices.length; k++) {\n\t\t\t\tIndexElement element = new IndexElement(cuts[k], i, attIndices.length - k - 1);\n\t\t\t\tif (attIndices[k] > p) {\n\t\t\t\t\tp = attIndices[k];\n\t\t\t\t}\n\t\t\t\tint attIdx = attIndices[k];\n\t\t\t\tif (!map.containsKey(attIdx)) {\n\t\t\t\t\tmap.put(attIdx, new ArrayList<IndexElement>());\n\t\t\t\t}\n\t\t\t\tmap.get(attIdx).add(element);\n\t\t\t}\n\t\t}\n\t\tp++;\n\t\tindexes = new Index[p];\n\t\tfor (int j = 0; j < p; j++) {\n\t\t\tList<IndexElement> elements = map.get(j);\n\t\t\tif (elements != null) {\n\t\t\t\tCollections.sort(elements);\n\t\t\t\tindexes[j] = new Index(elements.toArray(new IndexElement[elements.size()]));\n\t\t\t} else {\n\t\t\t\tindexes[j] = new Index(EMPTY_INDEX);\n\t\t\t}\n\t\t}\n\t}\n\t\n\t/**\n\t * Adds a decision table to the list.\n\t * \n\t * @param dt the decision table to add.\n\t */\n\tpublic void add(DecisionTable dt) {\n\t\tdtList.add(dt);\n\t}\n\t\n\t/**\n\t * Returns the table at the specified position in this list.\n\t * \n\t * @param index the index of the element to return.\n\t * @return the table at the specified position in this list.\n\t */\n\tpublic DecisionTable get(int index) {\n\t\treturn dtList.get(index);\n\t}\n\t\n\t/**\n\t * Removes the last tree.\n\t */\n\tpublic void removeLast() {\n\t\tif (dtList.size() > 0) {\n\t\t\tdtList.remove(dtList.size() - 1);\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the size of this boosted decision table list.\n\t * \n\t * @return the size of this boosted decision table list.\n\t */\n\tpublic int size() {\n\t\treturn dtList.size();\n\t}\n\t\n\t/**\n\t * Replaces the table at the specified position in this list with the new table.\n\t * \n\t * @param index the index of the element to replace.\n\t * @param dt the decision table to be stored at the specified position.\n\t */\n\tpublic void set(int index, DecisionTable dt) {\n\t\tdtList.set(index, dt);\n\t}\n\t\n\t/**\n\t * Regresses an instance.\n\t * \n\t * @param instance the instance to regress.\n\t * @return a regressed value.\n\t */\n\tpublic double regress(Instance instance) {\n\t\tdouble[] values = instance.getValues();\n\t\tlong[] predIndices = new long[dtList.size()];\n\t\tif (instance.isSparse()) {\n\t\t\tint[] indices = ((SparseVector) instance.getVector()).getIndices();\n\t\t\tfor (int j = 0; j < Math.min(values.length, indexes.length); j++) {\n\t\t\t\tdouble v = values[j];\n\t\t\t\tint idx = indices[j];\n\t\t\t\tindexes[idx].setPredIdx(predIndices, v);\n\t\t\t}\n\t\t} else {\n\t\t\tfor (int j = 0; j < Math.min(values.length, indexes.length); j++) {\n\t\t\t\tdouble v = values[j];\n\t\t\t\tindexes[j].setPredIdx(predIndices, v);\n\t\t\t}\n\t\t}\n\t\t\n\t\tdouble pred = 0;\n\t\tfor (int i = 0 ; i < predIndices.length; i++) {\n\t\t\tDecisionTable dt = dtList.get(i);\n\t\t\tpred += dt.regress(predIndices[i]);\n\t\t}\n\t\treturn pred;\n\t}\n\t\n\t@Override\n\tpublic BoostedDTables copy() {\n\t\treturn copy(true);\n\t}\n\t\n\t/**\n\t * Copies this object.\n\t * \n\t * @param copyIndexes {@code true} if the indexes are also copied;\n\t * @return this object.\n\t */\n\tpublic BoostedDTables copy(boolean copyIndexes) {\n\t\tBoostedDTables copy = new BoostedDTables();\n\t\tcopy.dtList = new ArrayList<>();\n\t\tfor (DecisionTable dt : dtList) {\n\t\t\tcopy.dtList.add(dt.copy());\n\t\t}\n\t\tif (copyIndexes) {\n\t\t\tcopy.indexes = new Index[this.indexes.length];\n\t\t\tfor (int i = 0; i < copy.indexes.length; i++) {\n\t\t\t\tcopy.indexes[i] = indexes[i].copy();\n\t\t\t}\n\t\t}\n\t\treturn copy;\n\t}\n\n\t/**\n\t * Reads in this boosted decision tables.\n\t * \n\t * @param in the reader.\n\t * @throws Exception\n\t */\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tint n = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tfor (int j = 0; j < n; j++) {\n\t\t\tString line = in.readLine();\n\t\t\tString predictorName = line.substring(1, line.length() - 1).split(\": \")[1];\n\t\t\tClass<?> clazz = Class.forName(predictorName);\n\t\t\tDecisionTable dt = (DecisionTable) clazz.getDeclaredConstructor().newInstance();\n\t\t\tdt.read(in);\n\t\t\tthis.dtList.add(dt);\n\n\t\t\tin.readLine();\n\t\t}\n\t\tbuildIndex();\n\t}\n\n\t/**\n\t * Writes this boosted decision tables.\n\t * \n\t * @param out the writer.\n\t * @throws Exception\n\t */\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Length: \" + dtList.size());\n\t\tfor (DecisionTable dt : dtList) {\n\t\t\tdt.write(out);\n\t\t\tout.println();\n\t\t}\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/BoostedRTrees.java",
    "content": "package mltk.predictor.tree.ensemble;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\nimport java.util.ArrayList;\n\nimport mltk.core.Instance;\nimport mltk.predictor.tree.RTree;\n\n/**\n * Class for boosted regression trees. This is a base class for BRT.\n * \n * @author Yin Lou\n * \n */\npublic class BoostedRTrees extends RTreeList {\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic BoostedRTrees() {\n\t\tsuper();\n\t}\n\n\t/**\n\t * Constructs a regression tree list of length n. By default each tree is null.\n\t * \n\t * @param n the length.\n\t */\n\tpublic BoostedRTrees(int n) {\n\t\ttrees = new ArrayList<>(n);\n\t\tfor (int i = 0; i < n; i++) {\n\t\t\ttrees.add(null);\n\t\t}\n\t}\n\n\t/**\n\t * Regresses an instance.\n\t * \n\t * @param instance the instance.\n\t * @return the regressed\n\t */\n\tpublic double regress(Instance instance) {\n\t\tdouble pred = 0;\n\t\tfor (RTree rt : trees) {\n\t\t\tpred += rt.regress(instance);\n\t\t}\n\t\treturn pred;\n\t}\n\n\t@Override\n\tpublic BoostedRTrees copy() {\n\t\tBoostedRTrees copy = new BoostedRTrees();\n\t\tfor (RTree rt : trees) {\n\t\t\tcopy.trees.add(rt.copy());\n\t\t}\n\t\treturn copy;\n\t}\n\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tint n = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\tfor (int j = 0; j < n; j++) {\n\t\t\tString line = in.readLine();\n\t\t\tString predictorName = line.substring(1, line.length() - 1).split(\": \")[1];\n\t\t\tClass<?> clazz = Class.forName(predictorName);\n\t\t\tRTree rt = (RTree) clazz.getDeclaredConstructor().newInstance();\n\t\t\trt.read(in);\n\t\t\tthis.trees.add(rt);\n\n\t\t\tin.readLine();\n\t\t}\n\t}\n\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Length: \" + trees.size());\n\t\tfor (RTree rt : trees) {\n\t\t\trt.write(out);\n\t\t\tout.println();\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/RTreeList.java",
    "content": "package mltk.predictor.tree.ensemble;\r\n\r\nimport java.util.ArrayList;\r\nimport java.util.Iterator;\r\nimport java.util.List;\r\n\r\nimport mltk.core.Copyable;\r\nimport mltk.predictor.tree.RTree;\r\n\r\n/**\r\n * Class for regression tree list.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class RTreeList implements Iterable<RTree>, Copyable<RTreeList> {\r\n\r\n\tprotected List<RTree> trees;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic RTreeList() {\r\n\t\ttrees = new ArrayList<>();\r\n\t}\r\n\t\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param capacity the capacity of this tree list.\r\n\t */\r\n\tpublic RTreeList(int capacity) {\r\n\t\ttrees = new ArrayList<>(capacity);\r\n\t}\r\n\r\n\t/**\r\n\t * Adds a regression tree to the list.\r\n\t * \r\n\t * @param tree the regression tree to add.\r\n\t */\r\n\tpublic void add(RTree tree) {\r\n\t\ttrees.add(tree);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic RTreeList copy() {\r\n\t\tRTreeList copy = new RTreeList();\r\n\t\tfor (RTree rt : trees) {\r\n\t\t\tcopy.trees.add(rt.copy());\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the tree at the specified position in this list.\r\n\t * \r\n\t * @param index the index of the element to return.\r\n\t * @return the tree at the specified position in this list.\r\n\t */\r\n\tpublic RTree get(int index) {\r\n\t\treturn trees.get(index);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic Iterator<RTree> iterator() {\r\n\t\treturn trees.iterator();\r\n\t}\r\n\r\n\t/**\r\n\t * Removes the last tree.\r\n\t */\r\n\tpublic void removeLast() {\r\n\t\tif (trees.size() > 0) {\r\n\t\t\ttrees.remove(trees.size() - 1);\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Replaces the tree at the specified position in this list with the new tree.\r\n\t * \r\n\t * @param index the index of the element to replace.\r\n\t * @param rt the regression tree to be stored at the specified position.\r\n\t */\r\n\tpublic void set(int index, RTree rt) {\r\n\t\ttrees.set(index, rt);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the size of this list.\r\n\t * \r\n\t * @return the size of this list.\r\n\t */\r\n\tpublic int size() {\r\n\t\treturn trees.size();\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/TreeEnsembleLearner.java",
    "content": "package mltk.predictor.tree.ensemble;\n\nimport mltk.predictor.HoldoutValidatedLearner;\nimport mltk.predictor.tree.TreeLearner;\n\n/**\n * Class for learning tree ensembles.\n * \n * @author Yin Lou\n *\n */\npublic abstract class TreeEnsembleLearner extends HoldoutValidatedLearner {\n\n\tprotected TreeLearner treeLearner;\n\t\n\tpublic TreeLearner getTreeLearner() {\n\t\treturn treeLearner;\n\t}\n\t\n\tpublic void setTreeLearner(TreeLearner treeLearner) {\n\t\tthis.treeLearner = treeLearner;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/ag/AdditiveGroves.java",
    "content": "package mltk.predictor.tree.ensemble.ag;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.Regressor;\r\nimport mltk.predictor.tree.RegressionTree;\r\n\r\n/**\r\n * Class for Additive Groves.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class AdditiveGroves implements Regressor {\r\n\r\n\tprotected List<RegressionTree[]> groves;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic AdditiveGroves() {\r\n\t\tgroves = new ArrayList<>();\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tint bn = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\tgroves = new ArrayList<>();\r\n\t\tfor (int i = 0; i < bn; i++) {\r\n\t\t\tint tn = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\t\tRegressionTree[] grove = new RegressionTree[tn];\r\n\t\t\tfor (int j = 0; j < tn; j++) {\r\n\t\t\t\tin.readLine();\r\n\t\t\t\tRegressionTree rt = new RegressionTree();\r\n\t\t\t\trt.read(in);\r\n\t\t\t\tgrove[i] = rt;\r\n\r\n\t\t\t\tin.readLine();\r\n\t\t\t}\r\n\t\t\tgroves.add(grove);\r\n\t\t\tin.readLine();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"Bagging: \" + groves.size());\r\n\t\tfor (RegressionTree[] grove : groves) {\r\n\t\t\tout.println(\"Size: \" + grove.length);\r\n\t\t\tfor (RegressionTree rt : grove) {\r\n\t\t\t\trt.write(out);\r\n\t\t\t\tout.println();\r\n\t\t\t}\r\n\t\t\tout.println();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\tif (groves.size() == 0) {\r\n\t\t\treturn 0;\r\n\t\t}\r\n\t\tdouble pred = 0;\r\n\t\tfor (RegressionTree[] grove : groves) {\r\n\t\t\tfor (RegressionTree rt : grove) {\r\n\t\t\t\tpred += rt.regress(instance);\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn pred / groves.size();\r\n\t}\r\n\r\n\t@Override\r\n\tpublic AdditiveGroves copy() {\r\n\t\tAdditiveGroves copy = new AdditiveGroves();\r\n\t\tfor (RegressionTree[] grove : groves) {\r\n\t\t\tRegressionTree[] copyGrove = new RegressionTree[grove.length];\r\n\t\t\tfor (int i = 0; i < copyGrove.length; i++) {\r\n\t\t\t\tcopyGrove[i] = grove[i].copy();\r\n\t\t\t}\r\n\t\t\tcopy.groves.add(copyGrove);\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/ag/AdditiveGrovesLearner.java",
    "content": "package mltk.predictor.tree.ensemble.ag;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerOptions;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.Sampling;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Learner;\nimport mltk.predictor.evaluation.AUC;\nimport mltk.predictor.evaluation.RMSE;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.predictor.tree.RegressionTree;\nimport mltk.predictor.tree.RegressionTreeLearner;\nimport mltk.predictor.tree.RegressionTreeLearner.Mode;\nimport mltk.util.OptimUtils;\nimport mltk.util.Random;\nimport mltk.util.tuple.IntPair;\n\n/**\n * Class for learning Additive Groves. This class currently only supports layered training.\n *\n * <p>\n * Reference:<br>\n * D. Sorokina, R. Caruana and M. Riedewald. Additive Groves of Regression Trees. In <i>Proceedings of the 18th European\n * Conference on Machine Learning (ECML)</i>, Warsaw, Poland, 2007.\n * </p>\n *\n * @author Yin Lou\n *\n */\npublic class AdditiveGrovesLearner extends Learner {\n\t\n\tstatic class Options extends LearnerOptions {\n\n\t\t@Argument(name = \"-v\", description = \"valid set path\", required = true)\n\t\tString validPath = null;\n\n\t\t@Argument(name = \"-o\", description = \"output model path\")\n\t\tString outputModelPath = null;\n\n\t\t@Argument(name = \"-e\", description = \"AUC (a), RMSE (r) (default: r)\")\n\t\tString metric = \"rmse\";\n\n\t\t@Argument(name = \"-b\", description = \"bagging iterations (default: 60)\")\n\t\tint baggingIters = 60;\n\n\t\t@Argument(name = \"-n\", description = \"number of trees in a grove (default: 6)\")\n\t\tint n = 6;\n\n\t\t@Argument(name = \"-a\", description = \"minimum alpha (default: 0.01)\")\n\t\tdouble a = 0.01;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t}\n\n\t/**\n\t * Trains an additive groves model.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.tree.ensemble.ag.AdditiveGrovesLearner\n\t * -t\ttrain set path\n\t * -v\tvalid set path\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-o]\toutput model path\n\t * [-e]\tAUC (a), RMSE (r) (default: r)\n\t * [-b]\tbagging iterations (default: 60)\n\t * [-n]\tnumber of trees in a grove (default: 6)\n\t * [-a]\tminimum alpha (default: 0.01)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(AdditiveGrovesLearner.class, opts);\n\t\tSimpleMetric metric = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\tif (\"rmse\".startsWith(opts.metric)) {\n\t\t\t\tmetric = new RMSE();\n\t\t\t} else if (\"auc\".startsWith(opts.metric)) {\n\t\t\t\tmetric = new AUC();\n\t\t\t}\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\n\t\tAdditiveGrovesLearner learner = new AdditiveGrovesLearner();\n\t\tlearner.setBaggingIters(opts.baggingIters);\n\t\tlearner.setNumTrees(opts.n);\n\t\tlearner.setMinAlpha(opts.a);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setVerbose(opts.verbose);\n\n\t\tlong start = System.currentTimeMillis();\n\t\tAdditiveGroves ag = learner.buildRegressor(trainSet, validSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(ag, opts.outputModelPath);\n\t\t}\n\t}\n\n\tclass PerformanceMatrix {\n\n\t\tSimpleMetric metric;\n\t\tdouble[][][] perf;\n\n\t\tPerformanceMatrix(int maxNumTrees, int numAlphas, int baggingIters, SimpleMetric metric) {\n\t\t\tperf = new double[maxNumTrees][numAlphas][baggingIters];\n\t\t\tthis.metric = metric;\n\t\t}\n\n\t\tvoid expand(int maxNumTrees, int numAlphas, int baggingIters) {\n\t\t\t// TODO make sure all dimensions are increasing only\n\t\t\tdouble[][][] newPerf = new double[maxNumTrees][numAlphas][baggingIters];\n\t\t\tfor (int i = 0; i < perf.length; i++) {\n\t\t\t\tdouble[][] t1Old = perf[i];\n\t\t\t\tdouble[][] t1New = newPerf[i];\n\t\t\t\tfor (int j = 0; j < t1Old.length; j++) {\n\t\t\t\t\tdouble[] t2Old = t1Old[j];\n\t\t\t\t\tdouble[] t2New = t1New[j];\n\t\t\t\t\tfor (int k = 0; k < t2Old.length; k++) {\n\t\t\t\t\t\tt2New[k] = t2Old[k];\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tperf = newPerf;\n\t\t}\n\n\t\tvoid eval(int t, int a, int b, double[] preds, double[] targets) {\n\t\t\tperf[t][a][b] = metric.eval(preds, targets);\n\t\t}\n\n\t\tIntPair getBestParameters() {\n\t\t\tint bestT = 0;\n\t\t\tint bestA = 0;\n\t\t\tdouble bestPerf = metric.worstValue();\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Perf Matrix:\");\n\t\t\t\tfor (int i = 0; i < perf.length; i++) {\n\t\t\t\t\tdouble[][] tOutter = perf[i];\n\t\t\t\t\tfor (int j = 0; j < tOutter.length; j++) {\n\t\t\t\t\t\tdouble[] tInner = tOutter[j];\n\t\t\t\t\t\tdouble p = tInner[tInner.length - 1];\n\t\t\t\t\t\tSystem.out.print(p + \" \");\n\t\t\t\t\t\tif (metric.isFirstBetter(p, bestPerf)) {\n\t\t\t\t\t\t\tbestT = i;\n\t\t\t\t\t\t\tbestA = j;\n\t\t\t\t\t\t\tbestPerf = p;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tSystem.out.println();\n\t\t\t\t}\n\t\t\t\tSystem.out.println(\"Best perf on validation set = \" + bestPerf);\n\t\t\t}\n\t\t\treturn new IntPair(bestT, bestA);\n\t\t}\n\n\t\t/**\n\t\t * Returns {@code true} if the bagging converges.\n\t\t *\n\t\t * @param t the number of trees.\n\t\t * @param a the index of alpha.\n\t\t * @return {@code true} if the bagging converges.\n\t\t */\n\t\tboolean analyzeBagging(int t, int a) {\n\t\t\treturn OptimUtils.isConverged(perf[t][a], metric.isLargerBetter());\n\t\t}\n\n\t}\n\n\tclass ModelMatrix {\n\n\t\tAdditiveGroves[][] groves;\n\n\t\tModelMatrix(int maxNumTrees, int numAlphas) {\n\t\t\tgroves = new AdditiveGroves[maxNumTrees][numAlphas];\n\t\t}\n\n\t\tvoid expand(int maxNumTrees, int numAlphas, int baggingIters) {\n\t\t\tAdditiveGroves[][] newGroves = new AdditiveGroves[maxNumTrees][numAlphas];\n\t\t\tfor (int i = 0; i < groves.length; i++) {\n\t\t\t\tfor (int j = 0; j < groves[i].length; j++) {\n\t\t\t\t\tnewGroves[i][j] = groves[i][j];\n\t\t\t\t}\n\t\t\t}\n\t\t\tgroves = newGroves;\n\t\t}\n\n\t\tvoid add(int t, int a, RegressionTree[] grove) {\n\t\t\tif (groves[t][a] == null) {\n\t\t\t\tgroves[t][a] = new AdditiveGroves();\n\t\t\t}\n\t\t\tgroves[t][a].groves.add(grove);\n\t\t}\n\n\t}\n\n\tclass PredictionMatrix {\n\n\t\tdouble[][][] sumPrediction;\n\t\tint n;\n\n\t\tPredictionMatrix(int tn, int an, int n) {\n\t\t\tsumPrediction = new double[tn][an][n];\n\t\t\tthis.n = n;\n\t\t}\n\n\t\tvoid expand(int tn, int an) {\n\t\t\tdouble[][][] newSumPrediction = new double[tn][an][n];\n\t\t\tfor (int t = 0; t < sumPrediction.length; t++) {\n\t\t\t\tdouble[][] tSrc = sumPrediction[t];\n\t\t\t\tdouble[][] tDes = newSumPrediction[t];\n\t\t\t\tfor (int a = 0; a < tSrc.length; a++) {\n\t\t\t\t\tSystem.arraycopy(tSrc[a], 0, tDes[a], 0, n);\n\t\t\t\t}\n\t\t\t}\n\t\t\tsumPrediction = newSumPrediction;\n\t\t}\n\n\t}\n\n\tprivate int bestNumTrees;\n\tprivate int bestBaggingIters;\n\tprivate double bestAlpha;\n\n\tprivate int numTrees;\n\tprivate int baggingIters;\n\tprivate double minAlpha;\n\tprivate SimpleMetric metric;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic AdditiveGrovesLearner() {\n\t\tverbose = false;\n\t\tnumTrees = 6;\n\t\tbaggingIters = 60;\n\t\tminAlpha = 0.01;\n\t\tmetric = new RMSE();\n\t}\n\n\t/**\n\t * Returns the metric.\n\t * \n\t * @return the metric.\n\t */\n\tpublic SimpleMetric getMetric() {\n\t\treturn metric;\n\t}\n\n\t/**\n\t * Sets the metric. Currently only support {@link mltk.predictor.evaluation.RMSE RMSE} and\n\t * {@link mltk.predictor.evaluation.AUC AUC}.\n\t * \n\t * @param metric the metric.\n\t */\n\tpublic void setMetric(SimpleMetric metric) {\n\t\tif (metric instanceof RMSE || metric instanceof AUC) {\n\t\t\tthis.metric = metric;\n\t\t}\n\t}\n\n\t/**\n\t * Returns the best number of trees in a grove from latest run.\n\t * \n\t * @return the best number of trees in a grove from latest run.\n\t */\n\tpublic int getBestNumTrees() {\n\t\treturn bestNumTrees;\n\t}\n\n\t/**\n\t * Returns the best bagging iterations from latest run.\n\t * \n\t * @return the best bagging iterations from latest run.\n\t */\n\tpublic int getBestBaggingIters() {\n\t\treturn bestBaggingIters;\n\t}\n\n\t/**\n\t * Returns the best alpha from latest run.\n\t * \n\t * @return the best alpha from latest run.\n\t */\n\tpublic double getBestAlpha() {\n\t\treturn bestAlpha;\n\t}\n\n\t/**\n\t * Returns the number of trees in a grove. The number of trees may be adjusted during the training.\n\t * \n\t * @return the number of trees in a grove.\n\t */\n\tpublic int getNumTrees() {\n\t\treturn numTrees;\n\t}\n\n\t/**\n\t * Sets the number of trees in a grove. The number of trees may be adjusted during the training.\n\t * \n\t * @param numTrees the number of trees in a grove.\n\t */\n\tpublic void setNumTrees(int numTrees) {\n\t\tthis.numTrees = numTrees;\n\t}\n\n\t/**\n\t * Returns the minimum alpha. The minimum alpha may be adjusted during the training.\n\t * \n\t * @return the minimum alpha.\n\t */\n\tpublic double getMinAlpha() {\n\t\treturn minAlpha;\n\t}\n\n\t/**\n\t * Sets the minimum alpha. The minimum alpha may be adjusted during the training.\n\t * \n\t * @param minAlpha the minimum alpha.\n\t */\n\tpublic void setMinAlpha(double minAlpha) {\n\t\tthis.minAlpha = minAlpha;\n\t}\n\n\t/**\n\t * Returns the number of bagging iterations. The number of bagging iterations may be adjusted during the training.\n\t *\n\t * @return the number of bagging iterations.\n\t */\n\tpublic int getBaggingIters() {\n\t\treturn baggingIters;\n\t}\n\n\t/**\n\t * Sets the number of bagging iterations. The number of bagging iterations may be adjusted during the training.\n\t *\n\t * @param baggingIters the bagging iterations.\n\t */\n\tpublic void setBaggingIters(int baggingIters) {\n\t\tthis.baggingIters = baggingIters;\n\t}\n\n\t/**\n\t * Builds additive groves.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @return a regressor.\n\t */\n\tpublic AdditiveGroves buildRegressor(Instances trainSet, Instances validSet) {\n\t\tint bn = baggingIters;\n\t\tint tn = numTrees;\n\t\tint an = 6;\n\t\tList<Double> alphas = new ArrayList<>();\n\t\tfor (int a = 0; a < an; a++) {\n\t\t\talphas.add(getAlpha(a));\n\t\t}\n\n\t\tint prevBN = 0;\n\t\tint prevTN = 0;\n\t\tint prevAN = 0;\n\n\t\t// Backup targets\n\t\tdouble[] targetTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\ttargetTrain[i] = trainSet.get(i).getTarget();\n\t\t}\n\t\tdouble[] targetValid = new double[validSet.size()];\n\t\tfor (int i = 0; i < targetValid.length; i++) {\n\t\t\ttargetValid[i] = validSet.get(i).getTarget();\n\t\t}\n\n\t\tPerformanceMatrix perfMatrix = new PerformanceMatrix(tn, an, bn, metric);\n\t\tModelMatrix modelMatrix = new ModelMatrix(tn, an);\n\t\tPredictionMatrix predMatrix = new PredictionMatrix(tn, an, validSet.size());\n\n\t\tIntPair bestParams = null;\n\n\t\tfor (;;) {\n\t\t\t// Running bagging iterations\n\t\t\trunLayeredTraining(trainSet, validSet, prevBN, bn, 0, tn, 0, an, alphas, perfMatrix, modelMatrix,\n\t\t\t\t\tpredMatrix, targetTrain, targetValid);\n\n\t\t\tbestParams = perfMatrix.getBestParameters();\n\t\t\tboolean converged = true;\n\n\t\t\tprevBN = bn;\n\t\t\tprevTN = tn;\n\t\t\tprevAN = an;\n\n\t\t\t// Expand alpha\n\t\t\tif (bestParams.v2 == an - 1 && alphas.get(an - 1) > 1.0 / trainSet.size()) {\n\t\t\t\tconverged = false;\n\t\t\t\tan += 3;\n\t\t\t\tfor (int a = prevAN; a < an; a++) {\n\t\t\t\t\talphas.add(getAlpha(a));\n\t\t\t\t}\n\t\t\t\tSystem.out.println(an);\n\n\t\t\t\tpredMatrix.expand(tn, an);\n\t\t\t\tperfMatrix.expand(tn, an, bn);\n\t\t\t\tmodelMatrix.expand(tn, an, bn);\n\n\t\t\t\trunLayeredTraining(trainSet, validSet, 0, bn, 0, tn, prevAN, an, alphas, perfMatrix, modelMatrix,\n\t\t\t\t\t\tpredMatrix, targetTrain, targetValid);\n\t\t\t}\n\n\t\t\t// Expand number of trees\n\t\t\tif (bestParams.v1 == tn - 1) {\n\t\t\t\tconverged = false;\n\t\t\t\ttn += 3;\n\n\t\t\t\tpredMatrix.expand(tn, an);\n\t\t\t\tperfMatrix.expand(tn, an, bn);\n\t\t\t\tmodelMatrix.expand(tn, an, bn);\n\n\t\t\t\trunLayeredTraining(trainSet, validSet, 0, bn, prevTN, tn, 0, an, alphas, perfMatrix, modelMatrix,\n\t\t\t\t\t\tpredMatrix, targetTrain, targetValid);\n\t\t\t}\n\n\t\t\t// Expand bagging\n\t\t\tif (!perfMatrix.analyzeBagging(bestParams.v1, bestParams.v2)) {\n\t\t\t\tconverged = false;\n\t\t\t\tbn += 40;\n\n\t\t\t\tpredMatrix.expand(tn, an);\n\t\t\t\tperfMatrix.expand(tn, an, bn);\n\t\t\t\tmodelMatrix.expand(tn, an, bn);\n\t\t\t}\n\n\t\t\tif (converged) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(targetTrain[i]);\n\t\t}\n\n\t\tSystem.out.println(\"Best model:\");\n\t\tSystem.out.println(\"Alpha = \" + alphas.get(bestParams.v2));\n\t\tSystem.out.println(\"N = \" + (bestParams.v1 + 1));\n\t\tSystem.out.println(\"b = \" + bn);\n\t\tbestBaggingIters = bn;\n\t\tbestNumTrees = bestParams.v1 + 1;\n\t\tbestAlpha = getAlpha(bestParams.v2);\n\t\treturn modelMatrix.groves[bestParams.v1][bestParams.v2];\n\t}\n\n\t/**\n\t * Builds additive groves using layered training.\n\t * \n\t * @param trainSet the training set.\n\t * @param baggingIters the number of bagging iterations.\n\t * @param numTrees the number of trees in a grove.\n\t * @param alpha the alpha.\n\t * @return a regressor.\n\t */\n\tpublic AdditiveGroves runLayeredTraining(Instances trainSet, int baggingIters, int numTrees, double alpha) {\n\t\tfinal int n = trainSet.size();\n\n\t\t// Backup targets\n\t\tdouble[] targetTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\ttargetTrain[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\tint bn = baggingIters;\n\t\tint tn = numTrees;\n\t\tint an = getAlphaIdx(alpha, trainSet.size()) + 1;\n\t\tAdditiveGroves ag = new AdditiveGroves();\n\t\tfor (int b = 0; b < bn; b++) {\n\t\t\t// The most recent predictions for regression trees\n\t\t\tdouble[][] rtPreds = new double[tn][n];\n\t\t\t// The most recent residuals\n\t\t\tdouble[] residualTrain = new double[n];\n\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\tresidualTrain[i] = targetTrain[i];\n\t\t\t}\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + (b + 1) + \" out of \" + bn);\n\t\t\t}\n\n\t\t\tfor (int a = 0; a < an; a++) {\n\t\t\t\tdouble currAlpha = getAlpha(a);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"\\tBuilding models with alpha = \" + currAlpha);\n\t\t\t\t}\n\n\t\t\t\tRegressionTree[] grove = new RegressionTree[tn];\n\t\t\t\tbackfit(trainSet, currAlpha, grove, rtPreds, residualTrain);\n\t\t\t\tag.groves.add(grove);\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(targetTrain[i]);\n\t\t}\n\n\t\treturn ag;\n\t}\n\n\t@Override\n\tpublic AdditiveGroves build(Instances instances) {\n\t\tInstances trainSet = new Instances(instances.getAttributes(), instances.getTargetAttribute());\n\t\tInstances validSet = new Instances(instances.getAttributes(), instances.getTargetAttribute());\n\t\tint nTrain = instances.size() / 5 * 4;\n\t\tfor (int i = 0; i < nTrain; i++) {\n\t\t\ttrainSet.add(instances.get(i));\n\t\t}\n\t\tfor (int i = nTrain; i < instances.size(); i++) {\n\t\t\tvalidSet.add(instances.get(i));\n\t\t}\n\t\treturn buildRegressor(trainSet, validSet);\n\t}\n\n\tprotected double getAlpha(int an) {\n\t\tdouble alpha = 1;\n\t\tif (an % 3 == 0) {\n\t\t\talpha = 5;\n\t\t} else if (an % 3 == 1) {\n\t\t\talpha = 2;\n\t\t}\n\t\tfor (int i = 0; i < an / 3 + 1; i++) {\n\t\t\talpha /= 10;\n\t\t}\n\t\treturn alpha;\n\t}\n\n\tprotected int getAlphaIdx(double alpha, int n) {\n\t\tint idx = 0;\n\t\tdouble min = 1.0 / n;\n\t\twhile (alpha < getAlpha(idx) && min < alpha) {\n\t\t\tidx++;\n\t\t}\n\t\treturn idx;\n\t}\n\n\tprotected void backfit(Instances trainSet, double alpha, RegressionTree[] grove, double[][] rtPreds,\n\t\t\tdouble[] residualTrain) {\n\t\tMap<Integer, Integer> bagIndices = new HashMap<>();\n\t\tList<Integer> oobIndices = new ArrayList<>();\n\t\tSampling.createBootstrapSample(trainSet, bagIndices, oobIndices);\n\t\tInstances bag = new Instances(trainSet.getAttributes(), trainSet.getTargetAttribute(), bagIndices.size());\n\t\tfor (Integer idx : bagIndices.keySet()) {\n\t\t\tint weight = bagIndices.get(idx);\n\t\t\tInstance instance = trainSet.get(idx).clone();\n\t\t\tinstance.setWeight(weight);\n\t\t\tbag.add(instance);\n\t\t}\n\n\t\tRegressionTreeLearner rtLearner = new RegressionTreeLearner();\n\t\trtLearner.setConstructionMode(Mode.ALPHA_LIMITED);\n\t\trtLearner.setAlpha(alpha);\n\n\t\tdouble prevRMSE = evalRMSE(oobIndices, residualTrain);\n\t\tfor (;;) {\n\t\t\tfor (int iter = 0; iter < grove.length; iter++) {\n\t\t\t\tint treeIdx = (iter + grove.length - 1) % grove.length;\n\n\t\t\t\tdouble[] treePreds = rtPreds[treeIdx];\n\t\t\t\tfor (int i = 0; i < residualTrain.length; i++) {\n\t\t\t\t\tresidualTrain[i] += treePreds[i];\n\t\t\t\t\ttrainSet.get(i).setTarget(residualTrain[i]);\n\t\t\t\t}\n\n\t\t\t\tRegressionTree rt = rtLearner.build(bag);\n\t\t\t\tgrove[treeIdx] = rt;\n\n\t\t\t\tfor (int i = 0; i < residualTrain.length; i++) {\n\t\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\t\ttreePreds[i] = pred;\n\t\t\t\t\tresidualTrain[i] -= pred;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tdouble currRMSE = evalRMSE(oobIndices, residualTrain);\n\t\t\tif (currRMSE == 0 || (prevRMSE - currRMSE) / prevRMSE <= 0.002) {\n\t\t\t\tbreak;\n\t\t\t} else {\n\t\t\t\tprevRMSE = currRMSE;\n\t\t\t}\n\t\t}\n\n\t}\n\n\tprotected double regress(RegressionTree[] trees, Instance instance) {\n\t\tdouble pred = 0;\n\t\tfor (RegressionTree rt : trees) {\n\t\t\tpred += rt.regress(instance);\n\t\t}\n\t\treturn pred;\n\t}\n\n\tprotected void runLayeredTraining(Instances trainSet, Instances validSet, int bStart, int bEnd, int tStart,\n\t\t\tint tEnd, int aStart, int aEnd, List<Double> alphas, PerformanceMatrix perfMatrix, ModelMatrix modelMatrix,\n\t\t\tPredictionMatrix predMatrix, double[] targetTrain, double[] targetValid) {\n\t\tfinal int n = trainSet.size();\n\t\tfinal int tLen = tEnd - tStart;\n\t\tdouble[] predictionValid = new double[validSet.size()];\n\n\t\tfor (int b = bStart; b < bEnd; b++) {\n\t\t\t// The most recent predictions for regression trees\n\t\t\tdouble[][][] rtPreds = new double[tLen][tEnd][n];\n\t\t\t// The most recent residuals\n\t\t\tdouble[][] residualTrain = new double[tLen][n];\n\t\t\tfor (int t = 0; t < tLen; t++) {\n\t\t\t\tdouble[] residual = residualTrain[t];\n\t\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\t\tresidual[i] = targetTrain[i];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (aStart != 0) {\n\t\t\t\tfor (int t = 0; t < tEnd; t++) {\n\t\t\t\t\tRegressionTree[] grove = modelMatrix.groves[t][aStart - 1].groves.get(b);\n\t\t\t\t\tupdate(trainSet, grove, rtPreds, residualTrain, t);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + (b + 1) + \" out of \" + bEnd);\n\t\t\t}\n\n\t\t\tfor (int a = aStart; a < aEnd; a++) {\n\t\t\t\tdouble alpha = alphas.get(a);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tSystem.out.println(\"\\tBuilding models with alpha = \" + alpha);\n\t\t\t\t}\n\n\t\t\t\tfor (int t = tStart; t < tEnd; t++) {\n\t\t\t\t\tint numTrees = t + 1;\n\t\t\t\t\tint tIdx = t - tStart;\n\n\t\t\t\t\tRegressionTree[] grove = new RegressionTree[numTrees];\n\t\t\t\t\tbackfit(trainSet, alpha, grove, rtPreds[tIdx], residualTrain[tIdx]);\n\t\t\t\t\tmodelMatrix.add(t, a, grove);\n\n\t\t\t\t\t// Update predictions\n\t\t\t\t\tdouble[] sumPredictionValid = predMatrix.sumPrediction[t][a];\n\t\t\t\t\tfor (int i = 0; i < sumPredictionValid.length; i++) {\n\t\t\t\t\t\tsumPredictionValid[i] += regress(grove, validSet.get(i));\n\t\t\t\t\t\tpredictionValid[i] = sumPredictionValid[i] / (b + 1);\n\t\t\t\t\t}\n\t\t\t\t\tperfMatrix.eval(t, a, b, predictionValid, targetValid);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tprotected double evalRMSE(List<Integer> indices, double[] residual) {\n\t\tdouble rmse = 0;\n\t\tfor (Integer idx : indices) {\n\t\t\tdouble d = residual[idx];\n\t\t\trmse += d * d;\n\t\t}\n\t\trmse = Math.sqrt(rmse / indices.size());\n\t\treturn rmse;\n\t}\n\n\tprotected void update(Instances trainSet, RegressionTree[] grove, double[][][] rtPreds, double[][] residualTrain,\n\t\t\tint maxTN) {\n\t\tfor (int t = 0; t < maxTN; t++) {\n\t\t\tRegressionTree rt = grove[t];\n\t\t\tfor (int i = 0; i < trainSet.size(); i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\trtPreds[maxTN][t][i] = pred;\n\t\t\t\tresidualTrain[maxTN][i] -= pred;\n\t\t\t}\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/ag/package-info.java",
    "content": "/**\r\n * Provides algorithms for fitting additive groves (AGs).\r\n */\r\npackage mltk.predictor.tree.ensemble.ag;"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/BDT.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.ProbabilisticClassifier;\r\nimport mltk.predictor.Regressor;\r\nimport mltk.predictor.tree.ensemble.BoostedDTables;\r\nimport mltk.util.MathUtils;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.VectorUtils;\r\n\r\n/**\r\n * Class for boosted decision tables (BDTs).\r\n * \r\n * <p>\r\n * Reference:<br>\r\n * Y. Lou and M. Obukhov. BDT: Boosting Decision Tables for High Accuracy and Scoring Efficiency. In <i>Proceedings of the\r\n * 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD)</i>, Halifax, Nova Scotia, Canada, 2017.\r\n * </p>\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BDT implements ProbabilisticClassifier, Regressor {\r\n\r\n\tprotected BoostedDTables[] tables;\r\n\t\r\n\t/**\r\n\t * Constructs a BDT from a BRT object.\r\n\t * \r\n\t * @param brt the BRT object.\r\n\t * @return a BDT object.\r\n\t */\r\n\tpublic static BDT constructBDT(BRT brt) {\r\n\t\tint k = brt.trees.length;\r\n\t\tBDT bdt = new BDT();\r\n\t\tbdt.tables = new BoostedDTables[k];\r\n\t\tfor (int i = 0; i < bdt.tables.length; i++) {\r\n\t\t\tbdt.tables[i] = new BoostedDTables(brt.trees[i]);\r\n\t\t}\r\n\t\t\r\n\t\treturn bdt;\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic BDT() {\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param k the number of classes.\r\n\t */\r\n\tpublic BDT(int k) {\r\n\t\ttables = new BoostedDTables[k];\r\n\t\tfor (int i = 0; i < tables.length; i++) {\r\n\t\t\ttables[i] = new BoostedDTables();\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the table list for class k.\r\n\t * \r\n\t * @param k the class k.\r\n\t * @return the table list for class k.\r\n\t */\r\n\tpublic BoostedDTables getDecisionTreeList(int k) {\r\n\t\treturn tables[k];\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int classify(Instance instance) {\r\n\t\tdouble[] prob = predictProbabilities(instance);\r\n\t\treturn StatUtils.indexOfMax(prob);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tint k = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\ttables = new BoostedDTables[k];\r\n\t\tfor (int i = 0; i < tables.length; i++) {\r\n\t\t\tin.readLine();\r\n\t\t\ttables[i] = new BoostedDTables();\r\n\t\t\ttables[i].read(in);\r\n\t\t\t\r\n\t\t\tin.readLine();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"K: \" + tables.length);\r\n\t\tfor (BoostedDTables dtList : tables) {\r\n\t\t\tdtList.write(out);\r\n\t\t\tout.println();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\treturn tables[0].regress(instance);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] predictProbabilities(Instance instance) {\r\n\t\tif (tables.length == 1) {\r\n\t\t\tdouble[] prob = new double[2];\r\n\t\t\tdouble pred = regress(instance);\r\n\t\t\tprob[1] = MathUtils.sigmoid(pred);\r\n\t\t\tprob[0] = 1 - prob[1];\r\n\t\t\treturn prob;\r\n\t\t} else {\r\n\t\t\tdouble[] prob = new double[tables.length];\r\n\t\t\tdouble[] pred = new double[tables.length];\r\n\t\t\tfor (int i = 0; i < tables.length; i++) {\r\n\t\t\t\tpred[i] = tables[i].regress(instance);\r\n\t\t\t}\r\n\t\t\tdouble max = StatUtils.max(pred);\r\n\t\t\tdouble sum = 0;\r\n\t\t\tfor (int i = 0; i < prob.length; i++) {\r\n\t\t\t\tprob[i] = Math.exp(pred[i] - max);\r\n\t\t\t\tsum += prob[i];\r\n\t\t\t}\r\n\t\t\tVectorUtils.divide(prob, sum);\r\n\t\t\treturn prob;\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BDT copy() {\r\n\t\tBDT copy = new BDT(tables.length);\r\n\t\tfor (int i = 0; i < tables.length; i++) {\r\n\t\t\tcopy.tables[i] = tables[i].copy();\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/BRT.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\r\n\r\nimport java.io.BufferedReader;\r\nimport java.io.PrintWriter;\r\n\r\nimport mltk.core.Instance;\r\nimport mltk.predictor.ProbabilisticClassifier;\r\nimport mltk.predictor.Regressor;\r\nimport mltk.predictor.tree.RTree;\r\nimport mltk.predictor.tree.ensemble.BoostedRTrees;\r\nimport mltk.util.MathUtils;\r\nimport mltk.util.StatUtils;\r\nimport mltk.util.VectorUtils;\r\n\r\n/**\r\n * Class for boosted regression trees (BRTs).\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class BRT implements ProbabilisticClassifier, Regressor {\r\n\r\n\tprotected BoostedRTrees[] trees;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic BRT() {\r\n\r\n\t}\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param k the number of classes.\r\n\t */\r\n\tpublic BRT(int k) {\r\n\t\ttrees = new BoostedRTrees[k];\r\n\t\tfor (int i = 0; i < trees.length; i++) {\r\n\t\t\ttrees[i] = new BoostedRTrees();\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the tree list for class k.\r\n\t * \r\n\t * @param k the class k.\r\n\t * @return the tree list for class k.\r\n\t */\r\n\tpublic BoostedRTrees getRegressionTreeList(int k) {\r\n\t\treturn trees[k];\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int classify(Instance instance) {\r\n\t\tdouble[] prob = predictProbabilities(instance);\r\n\t\treturn StatUtils.indexOfMax(prob);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void read(BufferedReader in) throws Exception {\r\n\t\tint k = Integer.parseInt(in.readLine().split(\": \")[1]);\r\n\t\ttrees = new BoostedRTrees[k];\r\n\t\tfor (int i = 0; i < trees.length; i++) {\r\n\t\t\tin.readLine();\r\n\t\t\ttrees[i] = new BoostedRTrees();\r\n\t\t\ttrees[i].read(in);\r\n\t\t\t\r\n\t\t\tin.readLine();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic void write(PrintWriter out) throws Exception {\r\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\r\n\t\tout.println(\"K: \" + trees.length);\r\n\t\tfor (BoostedRTrees rtList : trees) {\r\n\t\t\trtList.write(out);\r\n\t\t\tout.println();\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double regress(Instance instance) {\r\n\t\treturn trees[0].regress(instance);\r\n\t}\r\n\r\n\t@Override\r\n\tpublic double[] predictProbabilities(Instance instance) {\r\n\t\tif (trees.length == 1) {\r\n\t\t\tdouble[] prob = new double[2];\r\n\t\t\tdouble pred = regress(instance);\r\n\t\t\tprob[1] = MathUtils.sigmoid(pred);\r\n\t\t\tprob[0] = 1 - prob[1];\r\n\t\t\treturn prob;\r\n\t\t} else {\r\n\t\t\tdouble[] prob = new double[trees.length];\r\n\t\t\tdouble[] pred = new double[trees.length];\r\n\t\t\tfor (int i = 0; i < trees.length; i++) {\r\n\t\t\t\tpred[i] = trees[i].regress(instance);\r\n\t\t\t}\r\n\t\t\tdouble max = StatUtils.max(pred);\r\n\t\t\tdouble sum = 0;\r\n\t\t\tfor (int i = 0; i < prob.length; i++) {\r\n\t\t\t\tprob[i] = Math.exp(pred[i] - max);\r\n\t\t\t\tsum += prob[i];\r\n\t\t\t}\r\n\t\t\tVectorUtils.divide(prob, sum);\r\n\t\t\treturn prob;\r\n\t\t}\r\n\t}\r\n\r\n\t@Override\r\n\tpublic BRT copy() {\r\n\t\tBRT copy = new BRT(trees.length);\r\n\t\tfor (int i = 0; i < trees.length; i++) {\r\n\t\t\tBoostedRTrees brts = trees[i];\r\n\t\t\tfor (RTree rt : brts) {\r\n\t\t\t\tcopy.trees[i].add(rt.copy());\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn copy;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/BRTLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport mltk.predictor.tree.RegressionTreeLearner.Mode;\nimport mltk.predictor.tree.ensemble.TreeEnsembleLearner;\n\n/**\n * Abstract class for boosted regression tree learner.\n * \n * @author Yin Lou\n *\n */\npublic abstract class BRTLearner extends TreeEnsembleLearner {\n\n\tprotected int maxNumIters;\n\tprotected double alpha;\n\tprotected double learningRate;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic BRTLearner() {\n\t\tverbose = false;\n\t\tmaxNumIters = 3500;\n\t\tlearningRate = 0.01;\n\t\talpha = 1;\n\t\t\n\t\tRobustRegressionTreeLearner rtLearner = new RobustRegressionTreeLearner();\n\t\trtLearner.setConstructionMode(Mode.NUM_LEAVES_LIMITED);\n\t\trtLearner.setMaxNumLeaves(100);\n\t\t\n\t\ttreeLearner = rtLearner;\n\t}\n\t\n\t/**\n\t * Returns the alpha.\n\t * \n\t * @return the alpha.\n\t */\n\tpublic double getAlpha() {\n\t\treturn alpha;\n\t}\n\t\n\t/**\n\t * Sets the alpha.\n\t * \n\t * @param alpha the parameter that controls the portion of the features to consider\n\t * for each boosting iteration.\n\t */\n\tpublic void setAlpha(double alpha) {\n\t\tthis.alpha = alpha;\n\t}\n\t\n\t/**\n\t * Returns the learning rate.\n\t * \n\t * @return the learning rate.\n\t */\n\tpublic double getLearningRate() {\n\t\treturn learningRate;\n\t}\n\n\t/**\n\t * Sets the learning rate.\n\t * \n\t * @param learningRate the learning rate.\n\t */\n\tpublic void setLearningRate(double learningRate) {\n\t\tthis.learningRate = learningRate;\n\t}\n\t\n\t/**\n\t * Returns the maximum number of iterations.\n\t * \n\t * @return the maximum number of iterations.\n\t */\n\tpublic int getMaxNumIters() {\n\t\treturn maxNumIters;\n\t}\n\t\n\t/**\n\t * Sets the maximum number of iterations.\n\t * \n\t * @param maxNumIters the maximum number of iterations.\n\t */\n\tpublic void setMaxNumIters(int maxNumIters) {\n\t\tthis.maxNumIters = maxNumIters;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/BRTUtils.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport mltk.predictor.tree.TreeLearner;\nimport mltk.predictor.tree.DecisionTableLearner;\nimport mltk.predictor.tree.RegressionTreeLearner;\n\nclass BRTUtils {\n\n\tpublic static TreeLearner parseTreeLearner(String baseLearner) {\n\t\tString[] data = baseLearner.split(\":\");\n\t\tif (data.length != 3) {\n\t\t\tthrow new IllegalArgumentException();\n\t\t}\n\t\tTreeLearner rtLearner = null;\n\t\tswitch(data[0]) {\n\t\t\tcase \"rt\":\n\t\t\t\trtLearner = new RegressionTreeLearner();\n\t\t\t\tbreak;\n\t\t\tcase \"rrt\":\n\t\t\t\trtLearner = new RobustRegressionTreeLearner();\n\t\t\t\tbreak;\n\t\t\tcase \"dt\":\n\t\t\t\trtLearner = new DecisionTableLearner();\n\t\t\t\tbreak;\n\t\t\tcase \"rdt\":\n\t\t\t\trtLearner = new RobustDecisionTableLearner();\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tSystem.err.println(\"Unknown regression tree learner: \" + data[0]);\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t}\n\t\trtLearner.setParameters(data[1] + \":\" + data[2]);\n\t\t\n\t\treturn rtLearner;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/LADBoostLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.HoldoutValidatedLearnerOptions;\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.MAE;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.predictor.tree.RegressionTree;\nimport mltk.predictor.tree.RegressionTreeLeaf;\nimport mltk.predictor.tree.TreeLearner;\nimport mltk.util.ArrayUtils;\nimport mltk.util.MathUtils;\nimport mltk.util.Permutation;\nimport mltk.util.Random;\n\n/**\n * Class for least-absolute-deviation boost learner.\n * \n * @author Yin Lou\n * \n */\npublic class LADBoostLearner extends BRTLearner {\n\t\n\tstatic class Options extends HoldoutValidatedLearnerOptions {\n\n\t\t@Argument(name = \"-b\", description = \"base learner (tree:mode:parameter) (default: rt:l:100)\")\n\t\tString baseLearner = \"rt:l:100\";\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t\t@Argument(name = \"-l\", description = \"learning rate (default: 0.01)\")\n\t\tdouble learningRate = 0.01;\n\n\t}\n\n\t/**\n\t * Trains a boosted tree ensemble using least-absolute-deviation as the objective function.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.tree.ensemble.brt.LADBoostLearner\n\t * -t\ttrain set path\n\t * -m\tmaximum number of iterations\n\t * [-v]\tvalid set path\n\t * [-e]\tevaluation metric (default: default metric of task)\n\t * [-S]\tconvergence criteria (default: -1)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-b]\tbase learner (tree:mode:parameter) (default: rt:l:100)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * [-l]\tlearning rate (default: 0.01)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(LADBoostLearner.class, opts);\n\t\tMetric metric = null;\n\t\tTreeLearner rtLearner = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\tif (opts.metric == null) {\n\t\t\t\tmetric = new MAE();\n\t\t\t} else {\n\t\t\t\tmetric = MetricFactory.getMetric(opts.metric);\n\t\t\t}\n\t\t\trtLearner = BRTUtils.parseTreeLearner(opts.baseLearner);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\t\t\n\t\tConvergenceTester ct = ConvergenceTester.parse(opts.cc);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tLADBoostLearner learner = new LADBoostLearner();\n\t\tlearner.setLearningRate(opts.learningRate);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setTreeLearner(rtLearner);\n\t\tlearner.setConvergenceTester(ct);\n\t\t\n\t\tif (opts.validPath != null) {\n\t\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\t\t\tlearner.setValidSet(validSet);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tBRT brt = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(brt, opts.outputModelPath);\n\t\t}\n\t}\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LADBoostLearner() {\n\t\t\n\t}\n\n\t@Override\n\tpublic BRT build(Instances instances) {\n\t\tif (metric == null) {\n\t\t\tmetric = new MAE();\n\t\t}\n\t\tif (validSet != null) {\n\t\t\treturn buildRegressor(instances, validSet, maxNumIters);\n\t\t} else {\n\t\t\treturn buildRegressor(instances, maxNumIters);\n\t\t}\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a regressor.\n\t */\n\tpublic BRT buildRegressor(Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tperm.permute();\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\t\tdouble intercept = ArrayUtils.getMedian(target);\n\t\tRegressionTree initialTree = new RegressionTree(new RegressionTreeLeaf(intercept));\n\t\tbrt.trees[0].add(initialTree);\n\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = target[i] - intercept;\n\t\t}\n\t\tdouble[] pValid = new double[validSet.size()];\n\t\tArrays.fill(pValid, intercept);\n\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare attributes\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\t// Prepare training set\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\ttrainSet.get(i).setTarget(MathUtils.sign(rTrain[i]));\n\t\t\t}\n\n\t\t\tRegressionTree rt = (RegressionTree) treeLearner.build(trainSet);\n\t\t\tbrt.trees[0].add(rt);\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Replace the leaf value by median\n\t\t\tMap<RegressionTreeLeaf, List<Integer>> map = new HashMap<>();\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tRegressionTreeLeaf leaf = rt.getLeafNode(trainSet.get(i));\n\t\t\t\tif (!map.containsKey(leaf)) {\n\t\t\t\t\tmap.put(leaf, new ArrayList<Integer>());\n\t\t\t\t}\n\t\t\t\tmap.get(leaf).add(i);\n\t\t\t}\n\t\t\tfor (Map.Entry<RegressionTreeLeaf, List<Integer>> entry : map.entrySet()) {\n\t\t\t\tRegressionTreeLeaf leaf = entry.getKey();\n\t\t\t\tList<Integer> list = entry.getValue();\n\t\t\t\tdouble[] values = new double[list.size()];\n\t\t\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\t\t\tvalues[i] = rTrain[list.get(i)];\n\t\t\t\t}\n\t\t\t\tdouble pred = ArrayUtils.getMedian(values) * learningRate;\n\t\t\t\tleaf.setPrediction(pred);\n\t\t\t}\n\n\t\t\t// Update predictions and residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\trTrain[i] -= pred;\n\t\t\t}\n\t\t\tfor (int i = 0; i < pValid.length; i++) {\n\t\t\t\tdouble pred = rt.regress(validSet.get(i));\n\t\t\t\tpValid[i] += pred;\n\t\t\t}\n\n\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\tct.add(measure);\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\t\tfor (int i = brt.trees[0].size() - 1; i > idx; i--) {\n\t\t\tbrt.trees[0].removeLast();\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a regressor.\n\t */\n\tpublic BRT buildRegressor(Instances trainSet, int maxNumIters) {\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tperm.permute();\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\t\tdouble intercept = ArrayUtils.getMedian(target);\n\t\tRegressionTree initialTree = new RegressionTree(new RegressionTreeLeaf(intercept));\n\t\tbrt.trees[0].add(initialTree);\n\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\tpTrain[i] = intercept;\n\t\t\trTrain[i] = target[i] - intercept;\n\t\t}\n\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare attributes\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\t// Prepare training set\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\ttrainSet.get(i).setTarget(MathUtils.sign(rTrain[i]));\n\t\t\t}\n\n\t\t\tRegressionTree rt = (RegressionTree) treeLearner.build(trainSet);\n\t\t\tbrt.trees[0].add(rt);\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Replace the leaf value by median\n\t\t\tMap<RegressionTreeLeaf, List<Integer>> map = new HashMap<>();\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tRegressionTreeLeaf leaf = rt.getLeafNode(trainSet.get(i));\n\t\t\t\tif (!map.containsKey(leaf)) {\n\t\t\t\t\tmap.put(leaf, new ArrayList<Integer>());\n\t\t\t\t}\n\t\t\t\tmap.get(leaf).add(i);\n\t\t\t}\n\t\t\tfor (Map.Entry<RegressionTreeLeaf, List<Integer>> entry : map.entrySet()) {\n\t\t\t\tRegressionTreeLeaf leaf = entry.getKey();\n\t\t\t\tList<Integer> list = entry.getValue();\n\t\t\t\tdouble[] values = new double[list.size()];\n\t\t\t\tfor (int i = 0; i < values.length; i++) {\n\t\t\t\t\tvalues[i] = rTrain[list.get(i)];\n\t\t\t\t}\n\t\t\t\tdouble pred = ArrayUtils.getMedian(values) * learningRate;\n\t\t\t\tleaf.setPrediction(pred);\n\t\t\t}\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\tpTrain[i] += pred;\n\t\t\t\trTrain[i] -= pred;\n\t\t\t}\n\n\t\t\tif (verbose) {\n\t\t\t\tdouble measure = simpleMetric.eval(pTrain, target);\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/LSBoostLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.util.Arrays;\nimport java.util.List;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.HoldoutValidatedLearnerOptions;\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.evaluation.RMSE;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.predictor.tree.RTree;\nimport mltk.predictor.tree.TreeLearner;\nimport mltk.util.Permutation;\nimport mltk.util.Random;\n\n/**\n * Class for least-squares boost learner.\n *\n * @author Yin Lou\n *\n */\npublic class LSBoostLearner extends BRTLearner {\n\t\n\tstatic class Options extends HoldoutValidatedLearnerOptions {\n\n\t\t@Argument(name = \"-b\", description = \"base learner (tree:mode:parameter) (default: rt:l:100)\")\n\t\tString baseLearner = \"rt:l:100\";\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t\t@Argument(name = \"-l\", description = \"learning rate (default: 0.01)\")\n\t\tdouble learningRate = 0.01;\n\n\t}\n\n\t/**\n\t * Trains a boosted tree ensemble using least square as the objective function.\n\t *\n\t * <pre>\n\t * Usage: mltk.predictor.tree.ensemble.brt.LSBoostLearner\n\t * -t\ttrain set path\n\t * -m\tmaximum number of iterations\n\t * [-v]\tvalid set path\n\t * [-e]\tevaluation metric (default: default metric of task)\n\t * [-S]\tconvergence criteria (default: -1) \n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-b]\tbase learner (tree:mode:parameter) (default: rt:l:100)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * [-l]\tlearning rate (default: 0.01)\n\t * </pre>\n\t *\n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(LSBoostLearner.class, opts);\n\t\tMetric metric = null;\n\t\tTreeLearner rtLearner = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\tif (opts.metric == null) {\n\t\t\t\tmetric = new RMSE();\n\t\t\t} else {\n\t\t\t\tmetric = MetricFactory.getMetric(opts.metric);\n\t\t\t}\n\t\t\trtLearner = BRTUtils.parseTreeLearner(opts.baseLearner);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\t\t\n\t\tConvergenceTester ct = ConvergenceTester.parse(opts.cc);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tLSBoostLearner learner = new LSBoostLearner();\n\t\tlearner.setLearningRate(opts.learningRate);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setTreeLearner(rtLearner);\n\t\tlearner.setConvergenceTester(ct);\n\t\t\n\t\tif (opts.validPath != null) {\n\t\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\t\t\tlearner.setValidSet(validSet);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tBRT brt = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0);\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(brt, opts.outputModelPath);\n\t\t}\n\t}\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic LSBoostLearner() {\n\t\n\t}\n\t\n\t@Override\n\tpublic BRT build(Instances instances) {\n\t\tif (metric == null) {\n\t\t\tmetric = new RMSE();\n\t\t}\n\t\tif (validSet != null) {\n\t\t\treturn buildRegressor(instances, validSet, maxNumIters);\n\t\t} else {\n\t\t\treturn buildRegressor(instances, maxNumIters);\n\t\t}\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t *\n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a regressor.\n\t */\n\tpublic BRT buildRegressor(Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tperm.permute();\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = target[i];\n\t\t}\n\t\tdouble[] pValid = new double[validSet.size()];\n\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare training set\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\t// Prepare training set\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t}\n\n\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\tif (learningRate != 1) {\n\t\t\t\trt.multiply(learningRate);\n\t\t\t}\n\t\t\tbrt.trees[0].add(rt);\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Update predictions and residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\trTrain[i] -= pred;\n\t\t\t}\n\t\t\tfor (int i = 0; i < pValid.length; i++) {\n\t\t\t\tdouble pred = rt.regress(validSet.get(i));\n\t\t\t\tpValid[i] += pred;\n\t\t\t}\n\n\t\t\tdouble measure = metric.eval(pValid, validSet);\n\t\t\tct.add(measure);\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\t\tfor (int i = brt.trees[0].size() - 1; i > idx; i--) {\n\t\t\tbrt.trees[0].removeLast();\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\n\t/**\n\t * Builds a regressor.\n\t *\n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a regressor.\n\t */\n\tpublic BRT buildRegressor(Instances trainSet, int maxNumIters) {\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tperm.permute();\n\n\t\t// Backup targets\n\t\tdouble[] target = new double[trainSet.size()];\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttarget[i] = trainSet.get(i).getTarget();\n\t\t}\n\n\t\tdouble[] pTrain = new double[trainSet.size()];\n\t\tdouble[] rTrain = new double[trainSet.size()];\n\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\trTrain[i] = target[i];\n\t\t}\n\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare training set\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\ttrainSet.get(i).setTarget(rTrain[i]);\n\t\t\t}\n\n\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\tif (learningRate != 1) {\n\t\t\t\trt.multiply(learningRate);\n\t\t\t}\n\t\t\tbrt.trees[0].add(rt);\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Update residuals\n\t\t\tfor (int i = 0; i < rTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\tpTrain[i] += pred;\n\t\t\t\trTrain[i] -= pred;\n\t\t\t}\n\n\t\t\tif (verbose) {\n\t\t\t\tdouble measure = simpleMetric.eval(pTrain, target);\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t}\n\n\t\t// Restore targets\n\t\tfor (int i = 0; i < target.length; i++) {\n\t\t\ttrainSet.get(i).setTarget(target[i]);\n\t\t}\n\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/LogitBoostLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.HoldoutValidatedLearnerOptions;\nimport mltk.core.Attribute;\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.NominalAttribute;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.evaluation.ConvergenceTester;\nimport mltk.predictor.evaluation.Error;\nimport mltk.predictor.evaluation.Metric;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.evaluation.SimpleMetric;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.predictor.tree.RTree;\nimport mltk.predictor.tree.TreeLearner;\nimport mltk.util.MathUtils;\nimport mltk.util.OptimUtils;\nimport mltk.util.Permutation;\nimport mltk.util.Random;\n\n/**\n * Class for logit boost learner.\n * \n * <p>\n * Reference:<br>\n * P. Li. Robust logitboost and adaptive base class (abc) logitboost. In <i>Proceedings of the 26th Conference on\n * Uncertainty in Artificial Intelligence (UAI)</i>, Catalina Island, CA, USA, 2010.\n * </p>\n * \n * @author Yin Lou\n * \n */\npublic class LogitBoostLearner extends BRTLearner {\n\t\n\tstatic class Options extends HoldoutValidatedLearnerOptions {\n\n\t\t@Argument(name = \"-b\", description = \"base learner (tree:mode:parameter) (default: rt:l:100)\")\n\t\tString baseLearner = \"rt:l:100\";\n\n\t\t@Argument(name = \"-m\", description = \"maximum number of iterations\", required = true)\n\t\tint maxNumIters = -1;\n\n\t\t@Argument(name = \"-s\", description = \"seed of the random number generator (default: 0)\")\n\t\tlong seed = 0L;\n\n\t\t@Argument(name = \"-l\", description = \"learning rate (default: 0.01)\")\n\t\tdouble learningRate = 0.01;\n\n\t}\n\n\t/**\n\t * Trains an additive logistic regression.\n\t * \n\t * <pre>\n\t * Usage: mltk.predictor.tree.ensemble.brt.LogitBoostLearner\n\t * -t\ttrain set path\n\t * -m\tmaximum number of iterations\n\t * [-v]\tvalid set path\n\t * [-e]\tevaluation metric (default: default metric of task)\n\t * [-S]\tconvergence criteria (default: -1)\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-b]\tbase learner (tree:mode:parameter) (default: rt:l:100)\n\t * [-s]\tseed of the random number generator (default: 0)\n\t * [-l]\tlearning rate (default: 0.01)\n\t * </pre>\n\t * \n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(LogitBoostLearner.class, opts);\n\t\tMetric metric = null;\n\t\tTreeLearner rtLearner = null;\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\tif (opts.metric == null) {\n\t\t\t\tmetric = new Error();\n\t\t\t} else {\n\t\t\t\tmetric = MetricFactory.getMetric(opts.metric);\n\t\t\t}\n\t\t\t// Using robust version of the base tree learner\n\t\t\topts.baseLearner = \"r\" + opts.baseLearner;\n\t\t\trtLearner = BRTUtils.parseTreeLearner(opts.baseLearner);\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tRandom.getInstance().setSeed(opts.seed);\n\t\t\n\t\tConvergenceTester ct = ConvergenceTester.parse(opts.cc);\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\tLogitBoostLearner learner = new LogitBoostLearner();\n\t\tlearner.setLearningRate(opts.learningRate);\n\t\tlearner.setMaxNumIters(opts.maxNumIters);\n\t\tlearner.setVerbose(opts.verbose);\n\t\tlearner.setMetric(metric);\n\t\tlearner.setTreeLearner(rtLearner);\n\t\tlearner.setConvergenceTester(ct);\n\t\t\n\t\tif (opts.validPath != null) {\n\t\t\tInstances validSet = InstancesReader.read(opts.attPath, opts.validPath);\n\t\t\tlearner.setValidSet(validSet);\n\t\t}\n\n\t\tlong start = System.currentTimeMillis();\n\t\tBRT brt = learner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0 + \" (s).\");\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(brt, opts.outputModelPath);\n\t\t}\n\t}\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic LogitBoostLearner() {\n\t\n\t}\n\t\n\t@Override\n\tpublic BRT build(Instances instances) {\n\t\tif (metric == null) {\n\t\t\tmetric = new Error();\n\t\t}\n\t\tif (validSet != null) {\n\t\t\treturn buildClassifier(instances, validSet, maxNumIters);\n\t\t} else {\n\t\t\treturn buildClassifier(instances, maxNumIters);\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a classifier.\n\t */\n\tpublic BRT buildBinaryClassifier(Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tif (clazz.getCardinality() != 2) {\n\t\t\tthrow new IllegalArgumentException(\"Only binary classification is accepted.\");\n\t\t}\n\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tif (alpha < 1) {\n\t\t\tperm.permute();\n\t\t}\n\n\t\t// Backup targets and weights\n\t\tdouble[] targetTrain = new double[trainSet.size()];\n\t\tdouble[] weightTrain = new double[targetTrain.length];\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\ttargetTrain[i] = instance.getTarget();\n\t\t\tweightTrain[i] = instance.getWeight();\n\t\t}\n\n\t\t// Initialization\n\t\tdouble[] predTrain = new double[targetTrain.length];\n\t\tdouble[] probTrain = new double[targetTrain.length];\n\t\tcomputeProbabilities(predTrain, probTrain);\n\t\tdouble[] rTrain = new double[targetTrain.length];\n\t\tOptimUtils.computePseudoResidual(predTrain, targetTrain, rTrain);\n\t\tdouble[] predValid = new double[validSet.size()];\n\n\t\t// Resets the convergence tester\n\t\tct.setMetric(metric);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare attributes\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\t\n\t\t\t// Prepare training set\n\t\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\tdouble prob = probTrain[i];\n\t\t\t\tdouble w = prob * (1 - prob);\n\t\t\t\tinstance.setTarget(rTrain[i] * weightTrain[i]);\n\t\t\t\tinstance.setWeight(w * weightTrain[i]);\n\t\t\t}\n\t\t\t\n\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\tif (learningRate != 1) {\n\t\t\t\trt.multiply(learningRate);\n\t\t\t}\n\t\t\tbrt.trees[0].add(rt);\n\t\t\t\n\t\t\tfor (int i = 0; i < predTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\tpredTrain[i] += pred;\n\t\t\t}\n\t\t\tfor (int i = 0; i < predValid.length; i++) {\n\t\t\t\tdouble pred = rt.regress(validSet.get(i));\n\t\t\t\tpredValid[i] += pred;\n\t\t\t}\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Update residuals and probabilities\n\t\t\tOptimUtils.computePseudoResidual(predTrain, targetTrain, rTrain);\n\t\t\tcomputeProbabilities(predTrain, probTrain);\n\t\t\t\n\t\t\tdouble measure = metric.eval(predValid, validSet);\n\t\t\tct.add(measure);\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t\tif (ct.isConverged()) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\t\n\t\t// Search the best model on validation set\n\t\tint idx = ct.getBestIndex();\n\t\tfor (int i = brt.trees[0].size() - 1; i > idx; i--) {\n\t\t\tbrt.trees[0].removeLast();\n\t\t}\n\n\t\t// Restore targets and weights\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\tinstance.setTarget(targetTrain[i]);\n\t\t\tinstance.setWeight(weightTrain[i]);\n\t\t}\n\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\t\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a classifier.\n\t */\n\tpublic BRT buildBinaryClassifier(Instances trainSet, int maxNumIters) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tif (clazz.getCardinality() != 2) {\n\t\t\tthrow new IllegalArgumentException(\"Only binary classification is accepted.\");\n\t\t}\n\t\tSimpleMetric simpleMetric = (SimpleMetric) metric;\n\n\t\tBRT brt = new BRT(1);\n\t\ttreeLearner.cache(trainSet);\n\n\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\tint limit = (int) (attributes.size() * alpha);\n\t\tint[] indices = new int[limit];\n\t\tPermutation perm = new Permutation(attributes.size());\n\t\tif (alpha < 1) {\n\t\t\tperm.permute();\n\t\t}\n\n\t\t// Backup targets and weights\n\t\tdouble[] targetTrain = new double[trainSet.size()];\n\t\tdouble[] weightTrain = new double[targetTrain.length];\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\ttargetTrain[i] = instance.getTarget();\n\t\t\tweightTrain[i] = instance.getWeight();\n\t\t}\n\n\t\t// Initialization\n\t\tdouble[] predTrain = new double[targetTrain.length];\n\t\tdouble[] probTrain = new double[targetTrain.length];\n\t\tcomputeProbabilities(predTrain, probTrain);\n\t\tdouble[] rTrain = new double[targetTrain.length];\n\t\tOptimUtils.computePseudoResidual(predTrain, targetTrain, rTrain);\n\n\t\tList<Double> measureList = new ArrayList<>(maxNumIters);\n\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t// Prepare attributes\n\t\t\tif (alpha < 1) {\n\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t}\n\t\t\t\tArrays.sort(indices);\n\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t}\n\t\t\t\n\t\t\t// Prepare training set\n\t\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\tdouble prob = probTrain[i];\n\t\t\t\tdouble w = prob * (1 - prob);\n\t\t\t\tinstance.setTarget(rTrain[i] * weightTrain[i]);\n\t\t\t\tinstance.setWeight(w * weightTrain[i]);\n\t\t\t}\n\t\t\t\n\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\tif (learningRate != 1) {\n\t\t\t\trt.multiply(learningRate);\n\t\t\t}\n\t\t\tbrt.trees[0].add(rt);\n\t\t\t\n\t\t\tfor (int i = 0; i < predTrain.length; i++) {\n\t\t\t\tdouble pred = rt.regress(trainSet.get(i));\n\t\t\t\tpredTrain[i] += pred;\n\t\t\t}\n\n\t\t\tif (alpha < 1) {\n\t\t\t\t// Restore attributes\n\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t}\n\n\t\t\t// Update residuals and probabilities\n\t\t\tOptimUtils.computePseudoResidual(predTrain, targetTrain, rTrain);\n\t\t\tcomputeProbabilities(predTrain, probTrain);\n\t\t\t\n\t\t\tdouble measure = simpleMetric.eval(predTrain, targetTrain);\n\t\t\tmeasureList.add(measure);\n\t\t\tif (verbose) {\n\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + measure);\n\t\t\t}\n\t\t}\n\t\t\n\t\t// Search the best model on validation set\n\t\tint idx = metric.searchBestMetricValueIndex(measureList);\n\t\tfor (int i = brt.trees[0].size() - 1; i > idx; i--) {\n\t\t\tbrt.trees[0].removeLast();\n\t\t}\n\n\t\t// Restore targets and weights\n\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\tInstance instance = trainSet.get(i);\n\t\t\tinstance.setTarget(targetTrain[i]);\n\t\t\tinstance.setWeight(weightTrain[i]);\n\t\t}\n\t\t\n\t\ttreeLearner.evictCache();\n\t\treturn brt;\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param validSet the validation set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a classifier.\n\t */\n\tpublic BRT buildClassifier(Instances trainSet, Instances validSet, int maxNumIters) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tfinal int numClasses = clazz.getCardinality();\n\t\tif (numClasses == 2) {\n\t\t\treturn buildBinaryClassifier(trainSet, validSet, maxNumIters);\n\t\t} else {\n\t\t\tSystem.err.println(\"Multiclass LogitBoost, only use mis-classification rate as metric now\");\n\t\t\tfinal double l = learningRate * (numClasses - 1.0) / numClasses;\n\n\t\t\tBRT brt = new BRT(numClasses);\n\t\t\ttreeLearner.cache(trainSet);\n\n\t\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\t\tint limit = (int) (attributes.size() * alpha);\n\t\t\tint[] indices = new int[limit];\n\t\t\tPermutation perm = new Permutation(attributes.size());\n\t\t\tif (alpha < 1) {\n\t\t\t\tperm.permute();\n\t\t\t}\n\n\t\t\t// Backup targets and weights\n\t\t\tdouble[] targetTrain = new double[trainSet.size()];\n\t\t\tdouble[] weightTrain = new double[targetTrain.length];\n\t\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\ttargetTrain[i] = instance.getTarget();\n\t\t\t\tweightTrain[i] = instance.getWeight();\n\t\t\t}\n\t\t\tdouble[] targetValid = new double[validSet.size()];\n\t\t\tfor (int i = 0; i < targetValid.length; i++) {\n\t\t\t\ttargetValid[i] = validSet.get(i).getTarget();\n\t\t\t}\n\n\t\t\t// Initialization\n\t\t\tdouble[][] predTrain = new double[numClasses][targetTrain.length];\n\t\t\tdouble[][] probTrain = new double[numClasses][targetTrain.length];\n\t\t\tint[][] rTrain = new int[numClasses][targetTrain.length];\n\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\tint[] rkTrain = rTrain[k];\n\t\t\t\tdouble[] probkTrain = probTrain[k];\n\t\t\t\tfor (int i = 0; i < rkTrain.length; i++) {\n\t\t\t\t\trkTrain[i] = MathUtils.indicator(targetTrain[i] == k);\n\t\t\t\t\tprobkTrain[i] = 1.0 / numClasses;\n\t\t\t\t}\n\t\t\t}\n\t\t\tdouble[][] predValid = new double[numClasses][validSet.size()];\n\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\t// Prepare attributes\n\t\t\t\tif (alpha < 1) {\n\t\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t\t}\n\t\t\t\t\tArrays.sort(indices);\n\t\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// Prepare training set\n\t\t\t\t\tint[] rkTrain = rTrain[k];\n\t\t\t\t\tdouble[] probkTrain = probTrain[k];\n\t\t\t\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\t\tdouble pk = probkTrain[i];\n\t\t\t\t\t\tdouble t = rkTrain[i] - pk;\n\t\t\t\t\t\tdouble w = pk * (1 - pk);\n\t\t\t\t\t\tinstance.setTarget(t * weightTrain[i]);\n\t\t\t\t\t\tinstance.setWeight(w * weightTrain[i]);\n\t\t\t\t\t}\n\n\t\t\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\t\t\trt.multiply(l);\n\t\t\t\t\tbrt.trees[k].add(rt);\n\n\t\t\t\t\tdouble[] predkTrain = predTrain[k];\n\t\t\t\t\tfor (int i = 0; i < predkTrain.length; i++) {\n\t\t\t\t\t\tdouble p = rt.regress(trainSet.get(i));\n\t\t\t\t\t\tpredkTrain[i] += p;\n\t\t\t\t\t}\n\n\t\t\t\t\tdouble[] predkValid = predValid[k];\n\t\t\t\t\tfor (int i = 0; i < predkValid.length; i++) {\n\t\t\t\t\t\tdouble p = rt.regress(validSet.get(i));\n\t\t\t\t\t\tpredkValid[i] += p;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (alpha < 1) {\n\t\t\t\t\t// Restore attributes\n\t\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t\t}\n\n\t\t\t\t// Update probabilities\n\t\t\t\tcomputeProbabilities(predTrain, probTrain);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tdouble error = 0;\n\t\t\t\t\tfor (int i = 0; i < targetValid.length; i++) {\n\t\t\t\t\t\tdouble p = 0;\n\t\t\t\t\t\tdouble max = Double.NEGATIVE_INFINITY;\n\t\t\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t\t\tif (predValid[k][i] > max) {\n\t\t\t\t\t\t\t\tmax = predValid[k][i];\n\t\t\t\t\t\t\t\tp = k;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif (p != targetValid[i]) {\n\t\t\t\t\t\t\terror++;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\terror /= targetValid.length;\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + error);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Restore targets and weights\n\t\t\tfor (int i = 0; i < targetTrain.length; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\tinstance.setTarget(targetTrain[i]);\n\t\t\t\tinstance.setWeight(weightTrain[i]);\n\t\t\t}\n\n\t\t\ttreeLearner.evictCache();\n\t\t\treturn brt;\n\t\t}\n\t}\n\n\t/**\n\t * Builds a classifier.\n\t * \n\t * @param trainSet the training set.\n\t * @param maxNumIters the maximum number of iterations.\n\t * @return a classifier.\n\t */\n\tpublic BRT buildClassifier(Instances trainSet, int maxNumIters) {\n\t\tAttribute classAttribute = trainSet.getTargetAttribute();\n\t\tif (classAttribute.getType() != Attribute.Type.NOMINAL) {\n\t\t\tthrow new IllegalArgumentException(\"Class attribute must be nominal.\");\n\t\t}\n\t\tNominalAttribute clazz = (NominalAttribute) classAttribute;\n\t\tfinal int numClasses = clazz.getCardinality();\n\t\tif (numClasses == 2) {\n\t\t\treturn buildBinaryClassifier(trainSet, maxNumIters);\n\t\t} else {\n\t\t\tfinal int n = trainSet.size();\n\t\t\tfinal double l = learningRate * (numClasses - 1.0) / numClasses;\n\n\t\t\tBRT brt = new BRT(numClasses);\n\t\t\ttreeLearner.cache(trainSet);\n\n\t\t\tList<Attribute> attributes = trainSet.getAttributes();\n\t\t\tint limit = (int) (attributes.size() * alpha);\n\t\t\tint[] indices = new int[limit];\n\t\t\tPermutation perm = new Permutation(attributes.size());\n\t\t\tif (alpha < 1) {\n\t\t\t\tperm.permute();\n\t\t\t}\n\n\t\t\t// Backup targets and weights\n\t\t\tdouble[] target = new double[n];\n\t\t\tdouble[] weight = new double[n];\n\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\ttarget[i] = instance.getTarget();\n\t\t\t\tweight[i] = instance.getWeight();\n\t\t\t}\n\n\t\t\t// Initialization\n\t\t\tdouble[][] predTrain = new double[numClasses][n];\n\t\t\tdouble[][] probTrain = new double[numClasses][n];\n\t\t\tint[][] rTrain = new int[numClasses][n];\n\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\tint[] rkTrain = rTrain[k];\n\t\t\t\tdouble[] probkTrain = probTrain[k];\n\t\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\t\trkTrain[i] = MathUtils.indicator(target[i] == k);\n\t\t\t\t\tprobkTrain[i] = 1.0 / numClasses;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor (int iter = 0; iter < maxNumIters; iter++) {\n\t\t\t\t// Prepare attributes\n\t\t\t\tif (alpha < 1) {\n\t\t\t\t\tint[] a = perm.getPermutation();\n\t\t\t\t\tfor (int i = 0; i < indices.length; i++) {\n\t\t\t\t\t\tindices[i] = a[i];\n\t\t\t\t\t}\n\t\t\t\t\tArrays.sort(indices);\n\t\t\t\t\tList<Attribute> attList = trainSet.getAttributes(indices);\n\t\t\t\t\ttrainSet.setAttributes(attList);\n\t\t\t\t}\n\n\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t// Prepare training set\n\t\t\t\t\tint[] rkTrain = rTrain[k];\n\t\t\t\t\tdouble[] probkTrain = probTrain[k];\n\t\t\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\t\t\tdouble pk = probkTrain[i];\n\t\t\t\t\t\tdouble t = rkTrain[i] - pk;\n\t\t\t\t\t\tdouble w = pk * (1 - pk);\n\t\t\t\t\t\tinstance.setTarget(t * weight[i]);\n\t\t\t\t\t\tinstance.setWeight(w * weight[i]);\n\t\t\t\t\t}\n\n\t\t\t\t\tRTree rt = (RTree) treeLearner.build(trainSet);\n\t\t\t\t\trt.multiply(l);\n\t\t\t\t\tbrt.trees[k].add(rt);\n\n\t\t\t\t\tdouble[] predkTrain = predTrain[k];\n\t\t\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\t\t\tdouble p = rt.regress(trainSet.get(i));\n\t\t\t\t\t\tpredkTrain[i] += p;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (alpha < 1) {\n\t\t\t\t\t// Restore attributes\n\t\t\t\t\ttrainSet.setAttributes(attributes);\n\t\t\t\t}\n\n\t\t\t\t// Update probabilities\n\t\t\t\tcomputeProbabilities(predTrain, probTrain);\n\n\t\t\t\tif (verbose) {\n\t\t\t\t\tdouble error = 0;\n\t\t\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\t\t\tdouble p = 0;\n\t\t\t\t\t\tdouble maxProb = -1;\n\t\t\t\t\t\tfor (int k = 0; k < numClasses; k++) {\n\t\t\t\t\t\t\tif (probTrain[k][i] > maxProb) {\n\t\t\t\t\t\t\t\tmaxProb = probTrain[k][i];\n\t\t\t\t\t\t\t\tp = k;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif (p != target[i]) {\n\t\t\t\t\t\t\terror++;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\terror /= n;\n\t\t\t\t\tSystem.out.println(\"Iteration \" + iter + \": \" + error);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Restore targets and weights\n\t\t\tfor (int i = 0; i < n; i++) {\n\t\t\t\tInstance instance = trainSet.get(i);\n\t\t\t\tinstance.setTarget(target[i]);\n\t\t\t\tinstance.setWeight(weight[i]);\n\t\t\t}\n\n\t\t\ttreeLearner.evictCache();\n\t\t\treturn brt;\n\t\t}\n\t}\n\t\n\t@Override\n\tpublic void setTreeLearner(TreeLearner treeLearner) {\n\t\tif (!treeLearner.isRobust()) {\n\t\t\tthrow new IllegalArgumentException(\"Only robust tree learners are accepted\");\n\t\t}\n\t\tthis.treeLearner = treeLearner;\n\t}\n\n\tprotected void computeProbabilities(double[] pred, double[] prob) {\n\t\tfor (int i = 0; i < pred.length; i++) {\n\t\t\tprob[i] = MathUtils.sigmoid(pred[i]);\n\t\t}\n\t}\n\t\n\tprotected void computeProbabilities(double[][] pred, double[][] prob) {\n\t\tfor (int i = 0; i < pred[0].length; i++) {\n\t\t\tdouble max = Double.NEGATIVE_INFINITY;\n\t\t\tfor (int k = 0; k < pred.length; k++) {\n\t\t\t\tif (max < pred[k][i]) {\n\t\t\t\t\tmax = pred[k][i];\n\t\t\t\t}\n\t\t\t}\n\t\t\tdouble sum = 0;\n\t\t\tfor (int k = 0; k < pred.length; k++) {\n\t\t\t\tdouble p = Math.exp(pred[k][i] - max);\n\t\t\t\tprob[k][i] = p;\n\t\t\t\tsum += p;\n\t\t\t}\n\t\t\tfor (int k = 0; k < pred.length; k++) {\n\t\t\t\tprob[k][i] /= sum;\n\t\t\t}\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/RobustDecisionTableLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.util.Collections;\nimport java.util.List;\n\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.predictor.tree.DecisionTableLearner;\nimport mltk.util.tuple.DoublePair;\nimport mltk.util.tuple.IntDoublePair;\n\n/**\n * Class for learning decision tables in LogitBoost algorithm.\n * \n * @author Yin Lou\n *\n */\npublic class RobustDecisionTableLearner extends DecisionTableLearner {\n\t\n\tprotected boolean getStats(Instances instances, double[] stats) {\n\t\tstats[0] = stats[1] = stats[2] = 0;\n\t\tif (instances.size() == 0) {\n\t\t\treturn true;\n\t\t}\n\t\tdouble firstTarget = instances.get(0).getTarget();\n\t\tboolean stdIs0 = true;\n\t\tfor (Instance instance : instances) {\n\t\t\tdouble weight = instance.getWeight();\n\t\t\tdouble target = instance.getTarget();\n\t\t\tstats[0] += weight;\n\t\t\t// The key difference is we do not use weighted sum.\n\t\t\tstats[1] += target;\n\t\t\tif (stdIs0 && target != firstTarget) {\n\t\t\t\tstdIs0 = false;\n\t\t\t}\n\t\t}\n\t\tstats[2] = stats[1] / stats[0];\n\t\tif (Double.isNaN(stats[2])) {\n\t\t\tstats[2] = 0;\n\t\t}\n\t\treturn stdIs0;\n\t}\n\n\tprotected void getHistogram(Instances instances, List<IntDoublePair> pairs, List<Double> uniqueValues, double w,\n\t\t\tdouble s, List<DoublePair> histogram) {\n\t\tif (pairs.size() > 0) {\n\t\t\tdouble lastValue = pairs.get(0).v2;\n\t\t\tdouble totalWeight = instances.get(pairs.get(0).v1).getWeight();\n\t\t\t// The key difference is we do not use weighted sum.\n\t\t\tdouble sum = instances.get(pairs.get(0).v1).getTarget();\n\n\t\t\tfor (int i = 1; i < pairs.size(); i++) {\n\t\t\t\tIntDoublePair pair = pairs.get(i);\n\t\t\t\tdouble value = pair.v2;\n\t\t\t\tdouble weight = instances.get(pairs.get(i).v1).getWeight();\n\t\t\t\tdouble resp = instances.get(pairs.get(i).v1).getTarget();\n\t\t\t\tif (value != lastValue) {\n\t\t\t\t\tuniqueValues.add(lastValue);\n\t\t\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t\t\t\tlastValue = value;\n\t\t\t\t\ttotalWeight = weight;\n\t\t\t\t\tsum = resp;\n\t\t\t\t} else {\n\t\t\t\t\ttotalWeight += weight;\n\t\t\t\t\tsum += resp;\n\t\t\t\t}\n\t\t\t}\n\t\t\tuniqueValues.add(lastValue);\n\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t}\n\n\t\tif (pairs.size() != instances.size()) {\n\t\t\t// Zero entries are present\n\t\t\tdouble sumWeight = 0;\n\t\t\tdouble sumTarget = 0;\n\t\t\tfor (DoublePair pair : histogram) {\n\t\t\t\tsumWeight += pair.v1;\n\t\t\t\tsumTarget += pair.v2;\n\t\t\t}\n\n\t\t\tdouble weightOnZero = w - sumWeight;\n\t\t\tdouble sumOnZero = s - sumTarget;\n\t\t\tint idx = Collections.binarySearch(uniqueValues, ZERO);\n\t\t\tif (idx < 0) {\n\t\t\t\t// This should always happen\n\t\t\t\tuniqueValues.add(-idx - 1, ZERO);\n\t\t\t\thistogram.add(-idx - 1, new DoublePair(weightOnZero, sumOnZero));\n\t\t\t}\n\t\t}\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/RobustRegressionTreeLearner.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.util.Collections;\nimport java.util.List;\n\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.predictor.tree.RegressionTreeLearner;\nimport mltk.util.tuple.DoublePair;\nimport mltk.util.tuple.IntDoublePair;\n\n/**\n * Class for learning regression trees in LogitBoost algorithm. The splitting criteria ignores the weights when\n * calculating sum of responses.\n * \n * @author Yin Lou\n *\n */\npublic class RobustRegressionTreeLearner extends RegressionTreeLearner {\n\t\n\tpublic boolean isRobust() {\n\t\treturn true;\n\t}\n\n\tprotected boolean getStats(Instances instances, double[] stats) {\n\t\tstats[0] = stats[1] = stats[2] = 0;\n\t\tif (instances.size() == 0) {\n\t\t\treturn true;\n\t\t}\n\t\tdouble firstTarget = instances.get(0).getTarget();\n\t\tboolean stdIs0 = true;\n\t\tfor (Instance instance : instances) {\n\t\t\tdouble weight = instance.getWeight();\n\t\t\tdouble target = instance.getTarget();\n\t\t\tstats[0] += weight;\n\t\t\t// The key difference is we do not use weighted sum.\n\t\t\tstats[1] += target;\n\t\t\tif (stdIs0 && target != firstTarget) {\n\t\t\t\tstdIs0 = false;\n\t\t\t}\n\t\t}\n\t\tstats[2] = stats[1] / stats[0];\n\t\tif (Double.isNaN(stats[2])) {\n\t\t\tstats[2] = 0;\n\t\t}\n\t\treturn stdIs0;\n\t}\n\n\tprotected void getHistogram(Instances instances, List<IntDoublePair> pairs, List<Double> uniqueValues, double w,\n\t\t\tdouble s, List<DoublePair> histogram) {\n\t\tif (pairs.size() > 0) {\n\t\t\tdouble lastValue = pairs.get(0).v2;\n\t\t\tdouble totalWeight = instances.get(pairs.get(0).v1).getWeight();\n\t\t\t// The key difference is we do not use weighted sum.\n\t\t\tdouble sum = instances.get(pairs.get(0).v1).getTarget();\n\n\t\t\tfor (int i = 1; i < pairs.size(); i++) {\n\t\t\t\tIntDoublePair pair = pairs.get(i);\n\t\t\t\tdouble value = pair.v2;\n\t\t\t\tdouble weight = instances.get(pairs.get(i).v1).getWeight();\n\t\t\t\tdouble resp = instances.get(pairs.get(i).v1).getTarget();\n\t\t\t\tif (value != lastValue) {\n\t\t\t\t\tuniqueValues.add(lastValue);\n\t\t\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t\t\t\tlastValue = value;\n\t\t\t\t\ttotalWeight = weight;\n\t\t\t\t\tsum = resp;\n\t\t\t\t} else {\n\t\t\t\t\ttotalWeight += weight;\n\t\t\t\t\tsum += resp;\n\t\t\t\t}\n\t\t\t}\n\t\t\tuniqueValues.add(lastValue);\n\t\t\thistogram.add(new DoublePair(totalWeight, sum));\n\t\t}\n\n\t\tif (pairs.size() != instances.size()) {\n\t\t\t// Zero entries are present\n\t\t\tdouble sumWeight = 0;\n\t\t\tdouble sumTarget = 0;\n\t\t\tfor (DoublePair pair : histogram) {\n\t\t\t\tsumWeight += pair.v1;\n\t\t\t\tsumTarget += pair.v2;\n\t\t\t}\n\n\t\t\tdouble weightOnZero = w - sumWeight;\n\t\t\tdouble sumOnZero = s - sumTarget;\n\t\t\tint idx = Collections.binarySearch(uniqueValues, ZERO);\n\t\t\tif (idx < 0) {\n\t\t\t\t// This should always happen\n\t\t\t\tuniqueValues.add(-idx - 1, ZERO);\n\t\t\t\thistogram.add(-idx - 1, new DoublePair(weightOnZero, sumOnZero));\n\t\t\t}\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/brt/package-info.java",
    "content": "/**\r\n * Provides algorithms for fitting boosted regression trees (BRTs).\r\n */\r\npackage mltk.predictor.tree.ensemble.brt;"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/package-info.java",
    "content": "/**\n * Provides algorithms for tree ensemble methods.\n */\npackage mltk.predictor.tree.ensemble;"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/rf/RandomForest.java",
    "content": "package mltk.predictor.tree.ensemble.rf;\n\nimport java.io.BufferedReader;\nimport java.io.PrintWriter;\n\nimport mltk.core.Instance;\nimport mltk.predictor.Regressor;\nimport mltk.predictor.tree.RTree;\nimport mltk.predictor.tree.RegressionTree;\nimport mltk.predictor.tree.ensemble.RTreeList;\n\n/**\n * Class for random forests.\n * \n * @author Yin Lou\n *\n */\npublic class RandomForest implements Regressor {\n\t\n\tprotected RTreeList rtList;\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic RandomForest() {\n\t\trtList = new RTreeList();\n\t}\n\t\n\t/**\n\t * Constructor.\n\t * \n\t * @param capacity the capacity of this random forest.\n\t */\n\tpublic RandomForest(int capacity) {\n\t\trtList = new RTreeList(capacity);\n\t}\n\n\t@Override\n\tpublic void read(BufferedReader in) throws Exception {\n\t\tint capacity = Integer.parseInt(in.readLine().split(\": \")[1]);\n\t\trtList = new RTreeList(capacity);\n\t\tin.readLine();\n\t\tfor (int i = 0; i < capacity; i++) {\n\t\t\tin.readLine();\n\t\t\tRegressionTree rt = new RegressionTree();\n\t\t\trt.read(in);\n\t\t\trtList.add(rt);\n\t\t\t\n\t\t\tin.readLine();\n\t\t}\n\t}\n\n\t@Override\n\tpublic void write(PrintWriter out) throws Exception {\n\t\tout.printf(\"[Predictor: %s]\\n\", this.getClass().getCanonicalName());\n\t\tout.println(\"Ensemble: \" + size());\n\t\tout.println();\n\t\tfor (RTree rt : rtList) {\n\t\t\trt.write(out);\n\t\t\tout.println();\n\t\t}\n\t}\n\n\t@Override\n\tpublic RandomForest copy() {\n\t\tRandomForest copy = new RandomForest(size());\n\t\tfor (RTree rt : rtList) {\n\t\t\tcopy.add(rt.copy());\n\t\t}\n\t\treturn copy;\n\t}\n\n\t@Override\n\tpublic double regress(Instance instance) {\n\t\tif (size() == 0) {\n\t\t\treturn 0.0;\n\t\t} else {\n\t\t\tdouble prediction = 0.0;\n\t\t\tfor (RTree rt : rtList) {\n\t\t\t\tprediction += rt.regress(instance);\n\t\t\t}\n\t\t\treturn prediction / size();\n\t\t}\n\t}\n\t\n\t/**\n\t * Adds a regression tree to the ensemble.\n\t * \n\t * @param rt the regression tree.\n\t */\n\tpublic void add(RTree rt) {\n\t\trtList.add(rt);\n\t}\n\t\n\t/**\n\t * Returns the tree at the specified position in this list.\n\t * \n\t * @param index the index of the element to return.\n\t * @return the tree at the specified position in this list.\n\t */\n\tpublic RTree get(int index) {\n\t\treturn rtList.get(index);\n\t}\n\t\n\t/**\n\t * Returns the list of regression trees.\n\t * \n\t * @return the list of regression trees.\n\t */\n\tpublic RTreeList getTreeList() {\n\t\treturn rtList;\n\t}\n\t\n\t/**\n\t * Returns the size of this random forest.\n\t * \n\t * @return the size of this random forest.\n\t */\n\tpublic int size() {\n\t\treturn rtList.size();\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/rf/RandomForestLearner.java",
    "content": "package mltk.predictor.tree.ensemble.rf;\n\nimport mltk.cmdline.Argument;\nimport mltk.cmdline.CmdLineParser;\nimport mltk.cmdline.options.LearnerOptions;\nimport mltk.core.Instances;\nimport mltk.core.Sampling;\nimport mltk.core.io.InstancesReader;\nimport mltk.predictor.Learner;\nimport mltk.predictor.io.PredictorWriter;\nimport mltk.predictor.tree.RegressionTreeLearner;\nimport mltk.predictor.tree.RegressionTreeLearner.Mode;\n\n/**\n * Class for learning random forests.\n * \n * @author Yin Lou\n *\n */\npublic class RandomForestLearner extends Learner {\n\n\tstatic class Options extends LearnerOptions {\n\n\t\t@Argument(name = \"-m\", description = \"construction mode:parameter. Construction mode can be alpha limited (a), depth limited (d), number of leaves limited (l) and minimum leaf size limited (s) (default: a:0.001)\")\n\t\tString mode = \"a:0.001\";\n\n\t\t@Argument(name = \"-f\", description = \"number of features to consider (default: 1/3 of total number of features)\")\n\t\tint numFeatures = -1;\n\n\t\t@Argument(name = \"-b\", description = \"bagging iterations (default: 100)\")\n\t\tint baggingIters = 100;\n\n\t}\n\n\t/**\n\t * Trains a random forest of regression trees.\n\t *\n\t * When bagging is turned off (b = 0), this procedure generates a single random regression tree. When the number of\n\t * features to consider is the number of total features, this procedure builds bagged tree.\n\t *\n\t * <pre>\n\t * Usage: mltk.predictor.tree.ensemble.rf.RandomForestLearner\n\t * -t\ttrain set path\n\t * [-r]\tattribute file path\n\t * [-o]\toutput model path\n\t * [-V]\tverbose (default: true)\n\t * [-m]\tconstruction mode:parameter. Construction mode can be alpha limited (a), depth limited (d), number of leaves limited (l) and minimum leaf size limited (s) (default: a:0.001)\n\t * [-f]\tnumber of features to consider\n\t * [-b]\tbagging iterations (default: 100)\n\t * </pre>\n\t *\n\t * @param args the command line arguments.\n\t * @throws Exception\n\t */\n\tpublic static void main(String[] args) throws Exception {\n\t\tOptions opts = new Options();\n\t\tCmdLineParser parser = new CmdLineParser(RandomForestLearner.class, opts);\n\t\tRandomRegressionTreeLearner rtLearner = new RandomRegressionTreeLearner();\n\t\ttry {\n\t\t\tparser.parse(args);\n\t\t\tString[] data = opts.mode.split(\":\");\n\t\t\tif (data.length != 2) {\n\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t\tswitch (data[0]) {\n\t\t\t\tcase \"a\":\n\t\t\t\t\trtLearner.setConstructionMode(Mode.ALPHA_LIMITED);\n\t\t\t\t\trtLearner.setAlpha(Double.parseDouble(data[1]));\n\t\t\t\t\tbreak;\n\t\t\t\tcase \"d\":\n\t\t\t\t\trtLearner.setConstructionMode(Mode.DEPTH_LIMITED);\n\t\t\t\t\trtLearner.setMaxDepth(Integer.parseInt(data[1]));\n\t\t\t\t\tbreak;\n\t\t\t\tcase \"l\":\n\t\t\t\t\trtLearner.setConstructionMode(Mode.NUM_LEAVES_LIMITED);\n\t\t\t\t\trtLearner.setMaxNumLeaves(Integer.parseInt(data[1]));\n\t\t\t\t\tbreak;\n\t\t\t\tcase \"s\":\n\t\t\t\t\trtLearner.setConstructionMode(Mode.MIN_LEAF_SIZE_LIMITED);\n\t\t\t\t\trtLearner.setMinLeafSize(Integer.parseInt(data[1]));\n\t\t\t\tdefault:\n\t\t\t\t\tthrow new IllegalArgumentException();\n\t\t\t}\n\t\t} catch (IllegalArgumentException e) {\n\t\t\tparser.printUsage();\n\t\t\tSystem.exit(1);\n\t\t}\n\n\t\tInstances trainSet = InstancesReader.read(opts.attPath, opts.trainPath);\n\n\t\trtLearner.setNumFeatures(opts.numFeatures);\n\t\tRandomForestLearner rfLearner = new RandomForestLearner();\n\t\trfLearner.setBaggingIterations(opts.baggingIters);\n\t\trfLearner.setRegressionTreeLearner(rtLearner);\n\t\trfLearner.setVerbose(opts.verbose);\n\t\t\n\t\tlong start = System.currentTimeMillis();\n\t\tRandomForest rf = rfLearner.build(trainSet);\n\t\tlong end = System.currentTimeMillis();\n\t\tSystem.out.println(\"Time: \" + (end - start) / 1000.0 + \" (s).\");\n\n\t\tif (opts.outputModelPath != null) {\n\t\t\tPredictorWriter.write(rf, opts.outputModelPath);\n\t\t}\n\t}\n\t\n\tprivate int baggingIters;\n\tprivate RegressionTreeLearner rtLearner;\n\n\t@Override\n\tpublic RandomForest build(Instances instances) {\n\t\t// Create bags\n\t\tInstances[] bags = Sampling.createBags(instances, baggingIters);\n\t\t\n\t\tRandomForest rf = new RandomForest(baggingIters);\n\t\tfor (Instances bag : bags) {\n\t\t\trf.add(rtLearner.build(bag));\n\t\t}\n\t\treturn rf;\n\t}\n\t\n\t/**\n\t * Constructor.\n\t */\n\tpublic RandomForestLearner() {\n\t\tverbose = false;\n\t\tbaggingIters = 100;\n\t\trtLearner = new RandomRegressionTreeLearner();\n\t}\n\t\n\t/**\n\t * Returns the number of bagging iterations.\n\t * \n\t * @return the number of bagging iterations.\n\t */\n\tpublic int getBaggingIterations() {\n\t\treturn baggingIters;\n\t}\n\n\t/**\n\t * Sets the number of bagging iterations.\n\t * \n\t * @param baggingIters the number of bagging iterations.\n\t */\n\tpublic void setBaggingIterations(int baggingIters) {\n\t\tthis.baggingIters = baggingIters;\n\t}\n\t\n\t/**\n\t * Returns the regression tree learner.\n\t * \n\t * @return the regression tree learner.\n\t */\n\tpublic RegressionTreeLearner getRegressionTreeLearner() {\n\t\treturn rtLearner;\n\t}\n\t\n\t/**\n\t * Sets the regression tree learner.\n\t * \n\t * @param rtLearner the regression tree learner.\n\t */\n\tpublic void setRegressionTreeLearner(RegressionTreeLearner rtLearner) {\n\t\tthis.rtLearner = rtLearner;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/rf/RandomRegressionTreeLearner.java",
    "content": "package mltk.predictor.tree.ensemble.rf;\r\n\r\nimport java.util.ArrayList;\r\nimport java.util.HashSet;\r\nimport java.util.List;\r\nimport java.util.Set;\r\n\r\nimport mltk.core.Attribute;\r\nimport mltk.core.Instances;\r\nimport mltk.predictor.tree.RegressionTree;\r\nimport mltk.predictor.tree.RegressionTreeLeaf;\r\nimport mltk.predictor.tree.RegressionTreeLearner;\r\nimport mltk.predictor.tree.TreeInteriorNode;\r\nimport mltk.predictor.tree.TreeNode;\r\nimport mltk.util.Permutation;\r\nimport mltk.util.Random;\r\nimport mltk.util.tuple.DoublePair;\r\nimport mltk.util.tuple.IntDoublePair;\r\n\r\n/**\r\n * Class for learning random regression trees.\r\n *\r\n * @author Yin Lou\r\n *\r\n */\r\npublic class RandomRegressionTreeLearner extends RegressionTreeLearner {\r\n\r\n\tprotected int numFeatures;\r\n\tprotected Permutation perm;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t */\r\n\tpublic RandomRegressionTreeLearner() {\r\n\t\tnumFeatures = -1;\r\n\t\talpha = 0.01;\r\n\t\tmode = Mode.ALPHA_LIMITED;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the maximum number of features to consider for each node.\r\n\t *\r\n\t * @return the maximum number of features to consider for each node.\r\n\t */\r\n\tpublic int getNumFeatures() {\r\n\t\treturn numFeatures;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the maximum number of features to consider for each node.\r\n\t *\r\n\t * @param numFeatures the new maximum number of features.\r\n\t */\r\n\tpublic void setNumFeatures(int numFeatures) {\r\n\t\tthis.numFeatures = numFeatures;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic RegressionTree build(Instances instances) {\r\n\t\tif (numFeatures < 0) {\r\n\t\t\tnumFeatures = instances.getAttributes().size() / 3;\r\n\t\t}\r\n\t\tif (perm == null || perm.size() != instances.getAttributes().size()) {\r\n\t\t\tperm = new Permutation(instances.getAttributes().size());\r\n\t\t}\r\n\t\tRegressionTree rt = null;\r\n\t\tswitch (mode) {\r\n\t\t\tcase ALPHA_LIMITED:\r\n\t\t\t\trt = buildAlphaLimitedTree(instances, alpha);\r\n\t\t\t\tbreak;\r\n\t\t\tcase NUM_LEAVES_LIMITED:\r\n\t\t\t\trt = buildNumLeafLimitedTree(instances, maxNumLeaves);\r\n\t\t\t\tbreak;\r\n\t\t\tcase DEPTH_LIMITED:\r\n\t\t\t\trt = buildDepthLimitedTree(instances, maxDepth);\r\n\t\t\t\tbreak;\r\n\t\t\tcase MIN_LEAF_SIZE_LIMITED:\r\n\t\t\t\trt = buildMinLeafSizeLimitedTree(instances, minLeafSize);\r\n\t\t\tdefault:\r\n\t\t\t\tbreak;\r\n\t\t}\r\n\t\treturn rt;\r\n\t}\r\n\r\n\tprotected TreeNode createNode(Dataset dataset, int limit, double[] stats) {\r\n\t\tboolean stdIs0 = getStats(dataset.instances, stats);\r\n\t\tfinal double totalWeights = stats[0];\r\n\t\tfinal double sum = stats[1];\r\n\t\tfinal double weightedMean = stats[2];\r\n\r\n\t\t// 1. Check basic leaf conditions\r\n\t\tif (stats[0] < limit || stdIs0) {\r\n\t\t\tRegressionTreeLeaf node = new RegressionTreeLeaf(weightedMean);\r\n\t\t\treturn node;\r\n\t\t}\r\n\r\n\t\t// 2. Find best split\r\n\t\tdouble bestEval = Double.POSITIVE_INFINITY;\r\n\t\tList<IntDoublePair> splits = new ArrayList<>();\r\n\t\tList<Attribute> attributes = dataset.instances.getAttributes();\r\n\t\tint[] a = perm.permute().getPermutation();\r\n\t\tSet<Integer> selected = new HashSet<>(numFeatures);\r\n\t\tfor (int i = 0; i < numFeatures; i++) {\r\n\t\t\tselected.add(a[i]);\r\n\t\t}\r\n\t\tfor (int j = 0; j < attributes.size(); j++) {\r\n\t\t\tint attIndex = attributes.get(j).getIndex();\r\n\t\t\tif (!selected.contains(j)) {\r\n\t\t\t\tcontinue;\r\n\t\t\t}\r\n\t\t\tString attName = attributes.get(j).getName();\r\n\t\t\tList<IntDoublePair> sortedList = dataset.sortedLists.get(attName);\r\n\t\t\tList<Double> uniqueValues = new ArrayList<>(sortedList.size());\r\n\t\t\tList<DoublePair> histogram = new ArrayList<>(sortedList.size());\r\n\t\t\tgetHistogram(dataset.instances, sortedList, uniqueValues, totalWeights, sum, histogram);\r\n\r\n\t\t\tif (uniqueValues.size() > 1) {\r\n\t\t\t\tDoublePair split = split(uniqueValues, histogram, totalWeights, sum);\r\n\t\t\t\tif (split.v2 <= bestEval) {\r\n\t\t\t\t\tIntDoublePair splitPoint = new IntDoublePair(attIndex, split.v1);\r\n\t\t\t\t\tif (split.v2 < bestEval) {\r\n\t\t\t\t\t\tsplits.clear();\r\n\t\t\t\t\t\tbestEval = split.v2;\r\n\t\t\t\t\t}\r\n\t\t\t\t\tsplits.add(splitPoint);\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\tif (bestEval < Double.POSITIVE_INFINITY) {\r\n\t\t\tRandom rand = Random.getInstance();\r\n\t\t\tIntDoublePair splitPoint = splits.get(rand.nextInt(splits.size()));\r\n\t\t\tint attIndex = splitPoint.v1;\r\n\t\t\tTreeNode node = new TreeInteriorNode(attIndex, splitPoint.v2);\r\n\t\t\tstats[3] = bestEval + totalWeights * weightedMean * weightedMean;\r\n\t\t\treturn node;\r\n\t\t} else {\r\n\t\t\tRegressionTreeLeaf node = new RegressionTreeLeaf(weightedMean);\r\n\t\t\treturn node;\r\n\t\t}\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/ensemble/rf/package-info.java",
    "content": "/**\n * Provides algorithms for fitting random forests (RFs).\n */\npackage mltk.predictor.tree.ensemble.rf;"
  },
  {
    "path": "src/main/java/mltk/predictor/tree/package-info.java",
    "content": "/**\n * Provides algorithms for tree-based methods.\n */\npackage mltk.predictor.tree;"
  },
  {
    "path": "src/main/java/mltk/util/ArrayUtils.java",
    "content": "package mltk.util;\n\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * Class for utility functions for arrays.\n * \n * @author Yin Lou\n * \n */\npublic class ArrayUtils {\n\t\n\t/**\n\t * Converts an integer list to int array.\n\t * \n\t * @param list the list.\n\t * @return an int array.\n\t */\n\tpublic static int[] toIntArray(List<Integer> list) {\n\t\tint[] a = new int[list.size()];\n\t\tfor (int i = 0; i < list.size(); i++) {\n\t\t\ta[i] = list.get(i);\n\t\t}\n\t\treturn a;\n\t}\n\t\n\t/**\n\t * Converts a double list to double array.\n\t * \n\t * @param list the list.\n\t * @return a double array.\n\t */\n\tpublic static double[] toDoubleArray(List<Double> list) {\n\t\tdouble[] a = new double[list.size()];\n\t\tfor (int i = 0; i < list.size(); i++) {\n\t\t\ta[i] = list.get(i);\n\t\t}\n\t\treturn a;\n\t}\n\t\n\t/**\n\t * Converts a double array to an int array.\n\t * \n\t * @param a the double array.\n\t * @return an int array.\n\t */\n\tpublic static int[] toIntArray(double[] a) {\n\t\tint[] b = new int[a.length];\n\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\tb[i] = (int) a[i];\n\t\t}\n\t\treturn b;\n\t}\n\t\n\t/**\n\t * Returns a string representation of the contents of the specified sub-array.\n\t * \n\t * @param a the array.\n\t * @param start the starting index (inclusive).\n\t * @param end the ending index (exclusive).\n\t * @return Returns a string representation of the contents of the specified sub-array.\n\t */\n\tpublic static String toString(double[] a, int start, int end) {\n\t\tStringBuilder sb = new StringBuilder();\n\t\tsb.append(\"[\").append(a[start]);\n\t\tfor (int i = start + 1; i < end; i++) {\n\t\t\tsb.append(\", \").append(a[i]);\n\t\t}\n\t\tsb.append(\"]\");\n\t\treturn sb.toString();\n\t}\n\n\t/**\n\t * Parses a double array from a string (default delimiter: \",\").\n\t * \n\t * @param str the string representation of a double array.\n\t * @return a double array.\n\t */\n\tpublic static double[] parseDoubleArray(String str) {\n\t\treturn parseDoubleArray(str, \",\");\n\t}\n\n\t/**\n\t * Parses a double array from a string.\n\t * \n\t * @param str the string representation of a double array.\n\t * @param delimiter the delimiter.\n\t * @return a double array.\n\t */\n\tpublic static double[] parseDoubleArray(String str, String delimiter) {\n\t\tif (str == null || str.equalsIgnoreCase(\"null\")) {\n\t\t\treturn null;\n\t\t}\n\t\tString[] data = str.substring(1, str.length() - 1).split(delimiter);\n\t\tdouble[] a = new double[data.length];\n\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\ta[i] = Double.parseDouble(data[i].trim());\n\t\t}\n\t\treturn a;\n\t}\n\n\t/**\n\t * Parses an int array from a string (default delimiter: \",\").\n\t * \n\t * @param str the string representation of an int array.\n\t * @return an int array.\n\t */\n\tpublic static int[] parseIntArray(String str) {\n\t\treturn parseIntArray(str, \",\");\n\t}\n\n\t/**\n\t * Parses an int array from a string.\n\t * \n\t * @param str the string representation of an int array.\n\t * @param delimiter the delimiter.\n\t * @return an int array.\n\t */\n\tpublic static int[] parseIntArray(String str, String delimiter) {\n\t\tif (str == null || str.equalsIgnoreCase(\"null\")) {\n\t\t\treturn null;\n\t\t}\n\t\tString[] data = str.substring(1, str.length() - 1).split(delimiter);\n\t\tint[] a = new int[data.length];\n\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\ta[i] = Integer.parseInt(data[i].trim());\n\t\t}\n\t\treturn a;\n\t}\n\t\n\t/**\n\t * Parses a long array from a string (default delimiter: \",\").\n\t * \n\t * @param str the string representation of a long array.\n\t * @return an long array.\n\t */\n\tpublic static long[] parseLongArray(String str) {\n\t\treturn parseLongArray(str, \",\");\n\t}\n\t\n\t/**\n\t * Parses a long array from a string.\n\t * \n\t * @param str the string representation of a long array.\n\t * @param delimiter the delimiter.\n\t * @return a long array.\n\t */\n\tpublic static long[] parseLongArray(String str, String delimiter) {\n\t\tif (str == null || str.equalsIgnoreCase(\"null\")) {\n\t\t\treturn null;\n\t\t}\n\t\tString[] data = str.substring(1, str.length() - 1).split(delimiter);\n\t\tlong[] a = new long[data.length];\n\t\tfor (int i = 0; i < a.length; i++) {\n\t\t\ta[i] = Long.parseLong(data[i].trim());\n\t\t}\n\t\treturn a;\n\t}\n\n\t/**\n\t * Returns {@code true} if the specified range of an array is constant c.\n\t * \n\t * @param a the array.\n\t * @param begin the index of first element (inclusive).\n\t * @param end the index of last element (exclusive).\n\t * @param c the constant to test.\n\t * @return {@code true} if the specified range of an array is constant c.\n\t */\n\tpublic static boolean isConstant(double[] a, int begin, int end, double c) {\n\t\tfor (int i = begin; i < end; i++) {\n\t\t\tif (!MathUtils.equals(a[i], c)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\t\treturn true;\n\t}\n\n\t/**\n\t * Returns {@code true} if the specified range of an array is constant c.\n\t * \n\t * @param a the array.\n\t * @param begin the index of first element (inclusive).\n\t * @param end the index of last element (exclusive).\n\t * @param c the constant to test.\n\t * @return {@code true} if the specified range of an array is constant c.\n\t */\n\tpublic static boolean isConstant(int[] a, int begin, int end, int c) {\n\t\tfor (int i = begin; i < end; i++) {\n\t\t\tif (a[i] != c) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\t\treturn true;\n\t}\n\n\t/**\n\t * Returns {@code true} if the specified range of an array is constant c.\n\t * \n\t * @param a the array.\n\t * @param begin the index of first element (inclusive).\n\t * @param end the index of last element (exclusive).\n\t * @param c the constant to test.\n\t * @return {@code true} if the specified range of an array is constant c.\n\t */\n\tpublic static boolean isConstant(byte[] a, int begin, int end, byte c) {\n\t\tfor (int i = begin; i < end; i++) {\n\t\t\tif (a[i] != c) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\t\treturn true;\n\t}\n\n\t/**\n\t * Returns the median of an array.\n\t * \n\t * @param a the array.\n\t * @return the median of an array.\n\t */\n\tpublic static double getMedian(double[] a) {\n\t\tif (a.length == 0) {\n\t\t\treturn 0;\n\t\t}\n\t\tdouble[] ary = Arrays.copyOf(a, a.length);\n\t\tArrays.sort(ary);\n\t\tint mid = ary.length / 2;\n\t\tif (ary.length % 2 == 1) {\n\t\t\treturn ary[mid];\n\t\t} else {\n\t\t\treturn (ary[mid - 1] + ary[mid]) / 2;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the index of the search key if it is contained in the array, otherwise returns the insertion point.\n\t * \n\t * @param a the array.\n\t * @param key the search key.\n\t * @return the index of the search key if it is contained in the array, otherwise returns the insertion point.\n\t */\n\tpublic static int findInsertionPoint(double[] a, double key) {\n\t\tint idx = Arrays.binarySearch(a, key);\n\t\tif (idx < 0) {\n\t\t\tidx = -idx - 1;\n\t\t}\n\t\treturn idx;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/Element.java",
    "content": "package mltk.util;\n\n/**\n * Class for weighted elements.\n * \n * @author Yin Lou\n * \n * @param <T> the type of the element.\n */\npublic class Element<T> implements Comparable<Element<T>> {\n\n\tpublic T element;\n\tpublic double weight;\n\n\t/**\n\t * Constructs a weighted element.\n\t * \n\t * @param element the element.\n\t * @param weight the weight.\n\t */\n\tpublic Element(T element, double weight) {\n\t\tthis.element = element;\n\t\tthis.weight = weight;\n\t}\n\n\t@Override\n\tpublic int compareTo(Element<T> e) {\n\t\treturn Double.compare(this.weight, e.weight);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/MathUtils.java",
    "content": "package mltk.util;\n\n/**\n * Class for utility functions for math.\n * \n * @author Yin Lou\n * \n */\npublic class MathUtils {\n\n\t/**\n\t * 1e-8\n\t */\n\tpublic static final double EPSILON = 1e-8;\n\t\n\t/**\n\t * log(2)\n\t */\n\tpublic static final double LOG2 = Math.log(2);\n\t\n\t/**\n\t * Returns {@code true} if two doubles are equal to within {@link mltk.util.MathUtils#EPSILON}.\n\t * \n\t * @param a the 1st number.\n\t * @param b the 2nd number.\n\t * @return {@code true} if two doubles are equal to within {@link mltk.util.MathUtils#EPSILON}.\n\t */\n\tpublic static boolean equals(double a, double b) {\n\t\treturn Math.abs(a - b) < EPSILON;\n\t}\n\t\n\t/**\n\t * Returns 1 if the input is true and 0 otherwise.\n\t * \n\t * @param b the input.\n\t * @return 1 if the input is true and 0 otherwise.\n\t */\n\tpublic static int indicator(boolean b) {\n\t\treturn b ? 1 : 0;\n\t}\n\n\t/**\n\t * Returns {@code true} if the first value is better.\n\t * \n\t * @param a the 1st value.\n\t * @param b the 2nd value.\n\t * @param isLargerBetter {@code true} if the first value is better.\n\t * @return {@code true} if the first value is better.\n\t */\n\tpublic static boolean isFirstBetter(double a, double b, boolean isLargerBetter) {\n\t\tif (isLargerBetter) {\n\t\t\treturn a > b;\n\t\t} else {\n\t\t\treturn a < b;\n\t\t}\n\t}\n\n\t/**\n\t * Returns {@code true} if the floating number is integer.\n\t * \n\t * @param v the floating number.\n\t * @return {@code true} if the floating number is integer.\n\t */\n\tpublic static boolean isInteger(double v) {\n\t\treturn (v % 1) == 0;\n\t}\n\n\t/**\n\t * Returns {@code true} if the floating number is zero.\n\t * \n\t * @param v the floating number.\n\t * @return {@code true} if the floating number is zero.\n\t */\n\tpublic static boolean isZero(double v) {\n\t\treturn Math.abs(v) < EPSILON;\n\t}\n\n\t/**\n\t * Returns the value of a sigmoid function.\n\t * \n\t * @param a the number.\n\t * @return the value of a sigmoid function.\n\t */\n\tpublic static double sigmoid(double a) {\n\t\treturn 1 / (1 + Math.exp(-a));\n\t}\n\t\n\t/**\n\t * Returns the sign of a number.\n\t * \n\t * @param a the number.\n\t * @return the sign of a number.\n\t */\n\tpublic static int sign(double a) {\n\t\tif (a < 0) {\n\t\t\treturn -1;\n\t\t} else if (a > 0) {\n\t\t\treturn 1;\n\t\t} else {\n\t\t\treturn 0;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the sign of a number.\n\t * \n\t * @param a the number.\n\t * @return the sign of a number.\n\t */\n\tpublic static int sign(int a) {\n\t\tif (a < 0) {\n\t\t\treturn -1;\n\t\t} else if (a > 0) {\n\t\t\treturn 1;\n\t\t} else {\n\t\t\treturn 0;\n\t\t}\n\t}\n\t\n\t/**\n\t * Performs division and returns default value when division by zero.\n\t * \n\t * @param a the numerator.\n\t * @param b the denominator.\n\t * @param dv the default value.\n\t * @return a / b or default value when division by zero.\n\t */\n\tpublic static double divide(double a, double b, double dv) {\n\t\treturn isZero(b) ? dv : a / b;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/OptimUtils.java",
    "content": "package mltk.util;\n\nimport java.util.List;\n\n/**\n * Class for utility functions for optimization.\n * \n * @author Yin Lou\n * \n */\npublic class OptimUtils {\n\t\n\t/**\n\t * Returns the gain for variance reduction. This method is mostly used\n\t * in tree learners.\n\t * \n\t * @param sum the sum of responses.\n\t * @param weight the total weight.\n\t * @return the gain for variance reduction.\n\t */\n\tpublic static double getGain(double sum, double weight) {\n\t\tif (weight < MathUtils.EPSILON) {\n\t\t\treturn 0;\n\t\t} else {\n\t\t\treturn sum / weight * sum;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the probability of being in positive class.\n\t * \n\t * @param pred the prediction.\n\t * @return the probability of being in positive class.\n\t */\n\tpublic static double getProbability(double pred) {\n\t\treturn 1.0 / (1.0 + Math.exp(-pred));\n\t}\n\n\t/**\n\t * Returns the residual.\n\t * \n\t * @param pred the prediction.\n\t * @param target the target.\n\t * @return the residual.\n\t */\n\tpublic static double getResidual(double pred, double target) {\n\t\treturn target - pred;\n\t}\n\n\t/**\n\t * Returns the pseudo residual.\n\t * \n\t * @param pred the prediction.\n\t * @param cls the class label.\n\t * @return the pseudo residual.\n\t */\n\tpublic static double getPseudoResidual(double pred, double cls) {\n\t\treturn cls - getProbability(pred);\n\t}\n\t\n\t/**\n\t * Computes the pseudo residuals.\n\t * \n\t * @param prediction the prediction array.\n\t * @param y the class label array.\n\t * @param residual the residual array.\n\t */\n\tpublic static void computePseudoResidual(double[] prediction, double[] y, double[] residual) {\n\t\tfor (int i = 0; i < residual.length; i++) {\n\t\t\tresidual[i] = getPseudoResidual(prediction[i], y[i]);\n\t\t}\n\t}\n\t\n\t/**\n\t * Computes the probabilities.\n\t * \n\t * @param pred the prediction array.\n\t * @param prob the probability array.\n\t */\n\tpublic static void computeProbabilities(double[] pred, double[] prob) {\n\t\tfor (int i = 0; i < pred.length; i++) {\n\t\t\tprob[i] = getProbability(pred[i]);\n\t\t}\n\t}\n\n\t/**\n\t * Computes the logistic loss for binary classification problems.\n\t * \n\t * @param pred the prediction.\n\t * @param cls the class label.\n\t * @return the logistic loss for binary classification problems.\n\t */\n\tpublic static double computeLogisticLoss(double pred, double cls) {\n\t\tif (cls == 1) {\n\t\t\treturn Math.log(1 + Math.exp(-pred));\n\t\t} else {\n\t\t\treturn Math.log(1 + Math.exp(pred));\n\t\t}\n\t}\n\t\n\t/**\n\t * Computes the logistic loss for binary classification problems.\n\t * \n\t * @param pred the prediction array.\n\t * @param y the class label array.\n\t * @return the logistic loss for binary classification problems.\n\t */\n\tpublic static double computeLogisticLoss(double[] pred, double[] y) {\n\t\tdouble loss = 0;\n\t\tfor (int i = 0; i < pred.length; i++) {\n\t\t\tloss += computeLogisticLoss(pred[i], y[i]);\n\t\t}\n\t\treturn loss / y.length;\n\t}\n\t\n\t/**\n\t * Computes the log loss (cross entropy) for binary classification problems.\n\t * \n\t * @param prob the probability.\n\t * @param y the class label.\n\t * @return the log loss.\n\t */\n\tpublic static double computeLogLoss(double prob, double y) {\n\t\treturn computeLogLoss(prob, y, false);\n\t}\n\t\n\t/**\n\t * Computes the log loss (cross entropy) for binary classification problems.\n\t * \n\t * @param p the input.\n\t * @param y the class label.\n\t * @param isRawScore {@code true} if the input is raw score.\n\t * @return the log loss.\n\t */\n\tpublic static double computeLogLoss(double p, double y, boolean isRawScore) {\n\t\tif (isRawScore) {\n\t\t\tp = MathUtils.sigmoid(p);\n\t\t}\n\t\tif (y == 1) {\n\t\t\treturn -Math.log(p);\n\t\t} else {\n\t\t\treturn -Math.log(1 - p);\n\t\t}\n\t}\n\t\n\t/**\n\t * Computes the log loss (cross entropy) for binary classification problems.\n\t * \n\t * @param prob the probabilities.\n\t * @param y the class label array.\n\t * @return the log loss.\n\t */\n\tpublic static double computeLogLoss(double[] prob, double[] y) {\n\t\treturn computeLogLoss(prob, y, false);\n\t}\n\t\n\t/**\n\t * Computes the log loss (cross entropy) for binary classification problems.\n\t * \n\t * @param p the input.\n\t * @param y the targets\n\t * @param isRawScore {@code true} if the input is raw score.\n\t * @return the log loss.\n\t */\n\tpublic static double computeLogLoss(double[] p, double[] y, boolean isRawScore) {\n\t\tdouble logLoss = 0;\n\t\tfor (int i = 0; i < p.length; i++) {\n\t\t\tlogLoss += computeLogLoss(p[i], y[i], isRawScore);\n\t\t}\n\t\treturn logLoss;\n\t}\n\n\t/**\n\t * Computes the quadratic loss for regression problems.\n\t * \n\t * @param residual the residual array.\n\t * @return the quadratic loss for regression problems.\n\t */\n\tpublic static double computeQuadraticLoss(double[] residual) {\n\t\treturn StatUtils.sumSq(residual) / (2 * residual.length);\n\t}\n\t\n\t/**\n\t * Returns gradient on the intercept in regression problems. Residuals will be updated accordingly.\n\t * \n\t * @param residual the residual array.\n\t * @return the fitted intercept.\n\t */\n\tpublic static double fitIntercept(double[] residual) {\n\t\tdouble delta = StatUtils.mean(residual);\n\t\tVectorUtils.subtract(residual, delta);\n\t\treturn delta;\n\t}\n\n\t/**\n\t * Returns gradient on the intercept in binary classification problems. Predictions and residuals will be updated accordingly.\n\t * \n\t * @param prediction the prediction array.\n\t * @param residual the residual array.\n\t * @param y the class label array.\n\t * @return the fitted intercept.\n\t */\n\tpublic static double fitIntercept(double[] prediction, double[] residual, double[] y) {\n\t\tdouble delta = 0;\n\t\t// Use Newton-Raphson's method to approximate\n\t\t// 1st derivative\n\t\tdouble eta = 0;\n\t\t// 2nd derivative\n\t\tdouble theta = 0;\n\t\tfor (int i = 0; i < prediction.length; i++) {\n\t\t\tdouble r = residual[i];\n\t\t\tdouble t = Math.abs(r);\n\t\t\teta += r;\n\t\t\ttheta += t * (1 - t);\n\t\t}\n\n\t\tif (Math.abs(theta) > MathUtils.EPSILON) {\n\t\t\tdelta = eta / theta;\n\t\t\t// Update predictions\n\t\t\tVectorUtils.add(prediction, delta);\n\t\t\tcomputePseudoResidual(prediction, y, residual);\n\t\t}\n\t\treturn delta;\n\t}\n\t\n\t/**\n\t * Returns {@code true} if the relative improvement is less than a threshold.\n\t * \n\t * @param prevLoss the previous loss.\n\t * @param currLoss the current loss.\n\t * @param epsilon the threshold.\n\t * @return {@code true} if the relative improvement is less than a threshold.\n\t */\n\tpublic static boolean isConverged(double prevLoss, double currLoss, double epsilon) {\n\t\tif (prevLoss < MathUtils.EPSILON) {\n\t\t\treturn true;\n\t\t} else {\n\t\t\treturn (prevLoss - currLoss) / prevLoss < epsilon;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns {@code true} if the array of metric values is converged.\n\t * \n\t * @param p an array of metric values.\n\t * @param isLargerBetter {@code true} if larger value is better.\n\t * @return {@code true} if the list of metric values is converged.\n\t */\n\tpublic static boolean isConverged(double[] p, boolean isLargerBetter) {\n\t\tfinal int bn = p.length;\n\t\tif (p.length <= 20) {\n\t\t\treturn false;\n\t\t}\n\n\t\tdouble bestPerf = p[bn - 1];\n\t\tdouble worstPerf = p[bn - 20];\n\t\tfor (int i = bn - 20; i < bn; i++) {\n\t\t\tif (MathUtils.isFirstBetter(p[i], bestPerf, isLargerBetter)) {\n\t\t\t\tbestPerf = p[i];\n\t\t\t}\n\t\t\tif (!MathUtils.isFirstBetter(p[i], worstPerf, isLargerBetter)) {\n\t\t\t\tworstPerf = p[i];\n\t\t\t}\n\t\t}\n\t\tdouble relMaxMin = Math.abs(worstPerf - bestPerf) / worstPerf;\n\t\tdouble relImprov;\n\t\tif (MathUtils.isFirstBetter(p[bn - 1], p[bn - 21], isLargerBetter)) {\n\t\t\trelImprov = Math.abs(p[bn - 21] - p[bn - 1]) / p[bn - 21];\n\t\t} else {\n\t\t\t// Overfitting\n\t\t\trelImprov = Double.NaN;\n\t\t}\n\t\treturn relMaxMin < 0.02 && (Double.isNaN(relImprov) || relImprov < 0.005);\n\t}\n\t\n\t/**\n\t * Returns {@code true} if the list of metric values is converged.\n\t * \n\t * @param list a list of metric values.\n\t * @param isLargerBetter {@code true} if larger value is better.\n\t * @return {@code true} if the list of metric values is converged.\n\t */\n\tpublic static boolean isConverged(List<Double> list, boolean isLargerBetter) {\n\t\tif (list.size() <= 20) {\n\t\t\treturn false;\n\t\t}\n\n\t\tfinal int bn = list.size();\n\t\tdouble bestPerf = list.get(bn - 1);\n\t\tdouble worstPerf = list.get(bn - 20);\n\t\tfor (int i = bn - 20; i < bn; i++) {\n\t\t\tif (MathUtils.isFirstBetter(list.get(i), bestPerf, isLargerBetter)) {\n\t\t\t\tbestPerf = list.get(i);\n\t\t\t}\n\t\t\tif (!MathUtils.isFirstBetter(list.get(i), worstPerf, isLargerBetter)) {\n\t\t\t\tworstPerf = list.get(i);\n\t\t\t}\n\t\t}\n\t\tdouble relMaxMin = Math.abs(worstPerf - bestPerf) / worstPerf;\n\t\tdouble relImprov;\n\t\tif (MathUtils.isFirstBetter(list.get(bn - 1), list.get(bn - 21), isLargerBetter)) {\n\t\t\trelImprov = Math.abs(list.get(bn - 21) - list.get(bn - 1)) / list.get(bn - 21);\n\t\t} else {\n\t\t\t// Overfitting\n\t\t\trelImprov = Double.NaN;\n\t\t}\n\t\treturn relMaxMin < 0.02 && (Double.isNaN(relImprov) || relImprov < 0.005);\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/Permutation.java",
    "content": "package mltk.util;\r\n\r\n/**\r\n * Class for handling permutation.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Permutation {\r\n\r\n\tprotected int[] a;\r\n\r\n\t/**\r\n\t * Initializes a permutation of length n.\r\n\t * \r\n\t * @param n the length of a permutation.\r\n\t */\r\n\tpublic Permutation(int n) {\r\n\t\ta = new int[n];\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ta[i] = i;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Randomly permutes this permutation.\r\n\t * \r\n\t * @return this permutation.\r\n\t */\r\n\tpublic Permutation permute() {\r\n\t\tfor (int i = a.length - 1; i > 0; i--) {\r\n\t\t\tint idx = Random.getInstance().nextInt(i + 1);\r\n\t\t\tint t = a[idx];\r\n\t\t\ta[idx] = a[i];\r\n\t\t\ta[i] = t;\r\n\t\t}\r\n\t\treturn this;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the size of this permutation.\r\n\t * \r\n\t * @return the size of this permutation.\r\n\t */\r\n\tpublic int size() {\r\n\t\treturn a.length;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the permutation.\r\n\t * \r\n\t * @return the permutation.\r\n\t */\r\n\tpublic int[] getPermutation() {\r\n\t\treturn a;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/Queue.java",
    "content": "package mltk.util;\n\nimport java.util.LinkedList;\n\n/**\n * Class for generic queues.\n * \n * @author Yin Lou\n * \n * @param <T> the type of the queue.\n */\npublic class Queue<T> {\n\n\tprotected LinkedList<T> list;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Queue() {\n\t\tlist = new LinkedList<T>();\n\t}\n\n\t/**\n\t * Inserts an item to the queue.\n\t * \n\t * @param item the item.\n\t */\n\tpublic void enqueue(T item) {\n\t\tlist.addLast(item);\n\t}\n\n\t/**\n\t * Removes the first element in the queue.\n\t * \n\t * @return the first element in the queue.\n\t */\n\tpublic T dequeue() {\n\t\tT item = list.getFirst();\n\t\tlist.removeFirst();\n\t\treturn item;\n\t}\n\n\t/**\n\t * Returns {@code true} if the queue is empty.\n\t * \n\t * @return {@code true} if the queue is empty.\n\t */\n\tpublic boolean isEmpty() {\n\t\treturn list.size() == 0;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/Random.java",
    "content": "package mltk.util;\r\n\r\n/**\r\n * Class for global random object.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class Random {\r\n\r\n\tprotected static Random instance = null;\r\n\tprotected java.util.Random rand;\r\n\r\n\tprotected Random() {\r\n\t\trand = new java.util.Random();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the random object.\r\n\t * \r\n\t * @return the singleton random object.\r\n\t */\r\n\tpublic static Random getInstance() {\r\n\t\tif (instance == null) {\r\n\t\t\tinstance = new Random();\r\n\t\t}\r\n\t\treturn instance;\r\n\t}\r\n\r\n\t/**\r\n\t * Sets the random seed.\r\n\t * \r\n\t * @param seed the random seed.\r\n\t */\r\n\tpublic void setSeed(long seed) {\r\n\t\trand.setSeed(seed);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>int</code> value from this random number generator's\r\n\t * sequence.\r\n\t * \r\n\t * @return a random integer.\r\n\t */\r\n\tpublic int nextInt() {\r\n\t\treturn rand.nextInt();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>int</code> value between 0 (inclusive) and n\r\n\t * (exclusive) from this random number generator's sequence.\r\n\t * \r\n\t * @param n the range.\r\n\t * @return a random integer in [0, n- 1].\r\n\t */\r\n\tpublic int nextInt(int n) {\r\n\t\treturn rand.nextInt(n);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>double</code> value between 0.0 and 1.0 from this\r\n\t * random number generator's sequence.\r\n\t * \r\n\t * @return a random <code>double</code> value.\r\n\t */\r\n\tpublic double nextDouble() {\r\n\t\treturn rand.nextDouble();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>float</code> value between 0.0 and 1.0 from this\r\n\t * random number generator's sequence.\r\n\t * \r\n\t * @return a random <code>float</code> value.\r\n\t */\r\n\tpublic float nextFloat() {\r\n\t\treturn rand.nextFloat();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, Gaussian (\"normally\") distributed <code>\r\n\t * double</code> value with mean 0.0 and standard deviation 1.0 from this random number generator's sequence.\r\n\t * \r\n\t * @return a random <code>double</code> value.\r\n\t */\r\n\tpublic double nextGaussian() {\r\n\t\treturn rand.nextGaussian();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>long</code> value from this random number generator's\r\n\t * sequence.\r\n\t * \r\n\t * @return a random <code>long</code> value.\r\n\t */\r\n\tpublic long nextLong() {\r\n\t\treturn rand.nextLong();\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the next pseudorandom, uniformly distributed <code>boolean</code> value from this random number\r\n\t * generator's sequence.\r\n\t * \r\n\t * @return a random <code>boolean</code> value.\r\n\t */\r\n\tpublic boolean nextBoolean() {\r\n\t\treturn rand.nextBoolean();\r\n\t}\r\n\r\n\t/**\r\n\t * Generates random bytes and places them into a user-supplied byte array.\r\n\t * \r\n\t * @param bytes the byte array to fill with random bytes.\r\n\t */\r\n\tpublic void nextBytes(byte[] bytes) {\r\n\t\trand.nextBytes(bytes);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the backend Java random object.\r\n\t * \r\n\t * @return the backend Java random object.\r\n\t */\r\n\tpublic java.util.Random getRandom() {\r\n\t\treturn rand;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/Stack.java",
    "content": "package mltk.util;\n\nimport java.util.ArrayList;\nimport java.util.EmptyStackException;\nimport java.util.List;\n\n/**\n * Class for generic stacks.\n * \n * @author Yin Lou\n * \n * @param <T> the type of this stack.\n */\npublic class Stack<T> {\n\n\tprotected List<T> list;\n\n\t/**\n\t * Constructor.\n\t */\n\tpublic Stack() {\n\t\tthis.list = new ArrayList<T>();\n\t}\n\n\t/**\n\t * Inserts an item into the stack.\n\t * \n\t * @param item the item.\n\t */\n\tpublic void push(T item) {\n\t\tlist.add(item);\n\t}\n\n\t/**\n\t * Looks at the object at the top of this stack without removing it from the stack.\n\t * \n\t * @return the top element in the stack.\n\t */\n\tpublic T peek() {\n\t\tif (list.size() == 0) {\n\t\t\tthrow new EmptyStackException();\n\t\t}\n\t\treturn list.get(list.size() - 1);\n\t}\n\n\t/**\n\t * Removes the object at the top of this stack and returns that object as the value of this function.\n\t * \n\t * @return the top element in the stack.\n\t */\n\tpublic T pop() {\n\t\tT item = peek();\n\t\tlist.remove(list.size() - 1);\n\t\treturn item;\n\t}\n\n\t/**\n\t * Returns {@code true} if the stack is empty.\n\t * \n\t * @return {@code true} if the stack is empty.\n\t */\n\tpublic boolean isEmpty() {\n\t\treturn list.size() == 0;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/StatUtils.java",
    "content": "package mltk.util;\n\n/**\n * Class for utility functions for computing statistics.\n * \n * @author Yin Lou\n * \n */\npublic class StatUtils {\n\t\n\t/**\n\t * Returns the maximum element in an array.\n\t * \n\t * @param a the array.\n\t * @return the maximum element in an array.\n\t */\n\tpublic static int max(int[] a) {\n\t\tint max = a[0];\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] > max) {\n\t\t\t\tmax = a[i];\n\t\t\t}\n\t\t}\n\t\treturn max;\n\t}\n\n\t/**\n\t * Returns the maximum element in an array.\n\t * \n\t * @param a the array.\n\t * @return the maximum element in an array.\n\t */\n\tpublic static double max(double[] a) {\n\t\tdouble max = a[0];\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] > max) {\n\t\t\t\tmax = a[i];\n\t\t\t}\n\t\t}\n\t\treturn max;\n\t}\n\t\n\t/**\n\t * Returns the index of maximum element.\n\t * \n\t * @param a the array.\n\t * @return the index of maximum element.\n\t */\n\tpublic static int indexOfMax(int[] a) {\n\t\tint max = a[0];\n\t\tint idx = 0;\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] > max) {\n\t\t\t\tmax = a[i];\n\t\t\t\tidx = i;\n\t\t\t}\n\t\t}\n\t\treturn idx;\n\t}\n\n\t/**\n\t * Returns the index of maximum element.\n\t * \n\t * @param a the array.\n\t * @return the index of maximum element.\n\t */\n\tpublic static int indexOfMax(double[] a) {\n\t\tdouble max = a[0];\n\t\tint idx = 0;\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] > max) {\n\t\t\t\tmax = a[i];\n\t\t\t\tidx = i;\n\t\t\t}\n\t\t}\n\t\treturn idx;\n\t}\n\t\n\t/**\n\t * Returns the minimum element in an array.\n\t * \n\t * @param a the array.\n\t * @return the minimum element in an array.\n\t */\n\tpublic static int min(int[] a) {\n\t\tint min = a[0];\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] < min) {\n\t\t\t\tmin = a[i];\n\t\t\t}\n\t\t}\n\t\treturn min;\n\t}\n\n\t/**\n\t * Returns the minimum element in an array.\n\t * \n\t * @param a the array.\n\t * @return the minimum element in an array.\n\t */\n\tpublic static double min(double[] a) {\n\t\tdouble min = a[0];\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] < min) {\n\t\t\t\tmin = a[i];\n\t\t\t}\n\t\t}\n\t\treturn min;\n\t}\n\t\n\t/**\n\t * Returns the index of minimum element.\n\t * \n\t * @param a the array.\n\t * @return the index of minimum element.\n\t */\n\tpublic static int indexOfMin(int[] a) {\n\t\tint min = a[0];\n\t\tint idx = 0;\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] < min) {\n\t\t\t\tmin = a[i];\n\t\t\t\tidx = i;\n\t\t\t}\n\t\t}\n\t\treturn idx;\n\t}\n\n\t/**\n\t * Returns the index of minimum element.\n\t * \n\t * @param a the array.\n\t * @return the index of minimum element.\n\t */\n\tpublic static int indexOfMin(double[] a) {\n\t\tdouble min = a[0];\n\t\tint idx = 0;\n\t\tfor (int i = 1; i < a.length; i++) {\n\t\t\tif (a[i] < min) {\n\t\t\t\tmin = a[i];\n\t\t\t\tidx = i;\n\t\t\t}\n\t\t}\n\t\treturn idx;\n\t}\n\n\t/**\n\t * Returns the sum of elements in an array.\n\t * \n\t * @param a the array.\n\t * @return the sum of elements in an array.\n\t */\n\tpublic static double sum(double[] a) {\n\t\tdouble sum = 0;\n\t\tfor (double v : a) {\n\t\t\tsum += v;\n\t\t}\n\t\treturn sum;\n\t}\n\n\t/**\n\t * Returns the sum of squares.\n\t * \n\t * @param a the array.\n\t * @return the sum of squares.\n\t */\n\tpublic static double sumSq(double[] a) {\n\t\treturn sumSq(a, 0, a.length);\n\t}\n\n\t/**\n\t * Returns the sum of squares within a specific range.\n\t * \n\t * @param a the array.\n\t * @param fromIndex the index of the first element (inclusive).\n\t * @param toIndex the index of the last element (exclusive).\n\t * @return the sum of squares.\n\t */\n\tpublic static double sumSq(double[] a, int fromIndex, int toIndex) {\n\t\tdouble sq = 0.0;\n\t\tfor (int i = fromIndex; i < toIndex; i++) {\n\t\t\tsq += a[i] * a[i];\n\t\t}\n\t\treturn sq;\n\t}\n\n\t/**\n\t * Returns the mean.\n\t * \n\t * @param a the array.\n\t * @return the mean.\n\t */\n\tpublic static double mean(double[] a) {\n\t\treturn mean(a, a.length);\n\t}\n\n\t/**\n\t * Returns the mean.\n\t * \n\t * @param a the array.\n\t * @param n the total number of elements.\n\t * @return the mean.\n\t */\n\tpublic static double mean(double[] a, int n) {\n\t\tdouble avg = 0.0;\n\t\tfor (double v : a) {\n\t\t\tavg += v;\n\t\t}\n\t\treturn avg / n;\n\t}\n\n\t/**\n\t * Returns the variance.\n\t * \n\t * @param a the array.\n\t * @return the variance.\n\t */\n\tpublic static double variance(double[] a) {\n\t\treturn variance(a, a.length);\n\t}\n\n\t/**\n\t * Returns the variance.\n\t * \n\t * @param a the array.\n\t * @param n the total number of elements.\n\t * @return the variance.\n\t */\n\tpublic static double variance(double[] a, int n) {\n\t\tdouble avg = mean(a, n);\n\t\tdouble sq = 0.0;\n\t\tfor (double v : a) {\n\t\t\tdouble d = v - avg;\n\t\t\tsq += d * d;\n\t\t}\n\t\treturn sq / (n - 1.0);\n\t}\n\n\t/**\n\t * Returns the standard variance.\n\t * \n\t * @param a the array.\n\t * @return the standard variance.\n\t */\n\tpublic static double sd(double[] a) {\n\t\treturn sd(a, a.length);\n\t}\n\n\t/**\n\t * Returns the standard variance.\n\t * \n\t * @param a the array.\n\t * @param n the total number of elements.\n\t * @return the standard variance.\n\t */\n\tpublic static double sd(double[] a, int n) {\n\t\treturn Math.sqrt(variance(a, n));\n\t}\n\n\t/**\n\t * Returns the root mean square.\n\t * \n\t * @param a the array.\n\t * @return the root mean square.\n\t */\n\tpublic static double rms(double[] a) {\n\t\tdouble rms = 0.0;\n\t\tfor (double v : a) {\n\t\t\trms += v * v;\n\t\t}\n\t\trms /= a.length;\n\t\treturn Math.sqrt(rms);\n\t}\n\t\n\t/**\n\t * Returns the mean absolute deviation around a central point.\n\t * \n\t * @param a the array.\n\t * @param centralPoint the central point.\n\t * @return the mean absolute deviation around a central point.\n\t */\n\tpublic static double mad(double[] a, double centralPoint) {\n\t\tdouble mad = 0.0;\n\t\tfor (double v : a) {\n\t\t\tmad += Math.abs(v - centralPoint);\n\t\t}\n\t\treturn mad / a.length;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/UFSets.java",
    "content": "package mltk.util;\n\n/**\n * Class for union-find sets.\n * \n * @author Yin Lou\n *\n */\npublic class UFSets {\n\t\n\tprivate int[] parent;\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param size the size.\n\t */\n\tpublic UFSets(int size) {\n\t\tparent = new int[size + 1];\n\t\tfor (int i = 0; i < parent.length; i++) {\n\t\t\tparent[i] = -1;\n\t\t}\n\t}\n\t\n\t/**\n\t * Unions two sets.\n\t * \n\t * @param root1 the root for the 1st set.\n\t * @param root2 the root for the 2nd set.\n\t */\n\tpublic void union(int root1, int root2) {\n\t\tint temp = parent[root1] + parent[root2];\n\t\tif (parent[root1] < parent[root2]) {\n\t\t\tparent[root2] = root1;\n\t\t\tparent[root1] = temp;\n\t\t} else {\n\t\t\tparent[root1] = root2;\n\t\t\tparent[root2] = temp;\n\t\t}\n\t}\n\t\n\t/**\n\t * Returns the root of the set that contains the search key.\n\t * \n\t * @param i the search key.\n\t * @return the root of the set that contains the search key.\n\t */\n\tpublic int find(int i) {\n\t\tint j;\n\t\tfor (j = i; parent[j] >= 0; j = parent[j]);\n\t\twhile (i != j) {\n\t\t\tint temp = parent[i];\n\t\t\tparent[i] = j;\n\t\t\ti = temp;\n\t\t}\n\t\treturn j;\n\t}\n\t\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/VectorUtils.java",
    "content": "package mltk.util;\r\n\r\nimport mltk.core.DenseVector;\r\nimport mltk.core.SparseVector;\r\nimport mltk.core.Vector;\r\n\r\n/**\r\n * Class for utility functions for real vectors.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class VectorUtils {\r\n\r\n\t/**\r\n\t * Adds a constant to all elements in the array.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @param v the constant.\r\n\t */\r\n\tpublic static void add(double[] a, double v) {\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ta[i] += v;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Subtracts a constant from all elements in the array.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @param v the constant.\r\n\t */\r\n\tpublic static void subtract(double[] a, double v) {\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ta[i] -= v;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Multiplies a constant to all elements in the array.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @param v the constant.\r\n\t */\r\n\tpublic static void multiply(double[] a, double v) {\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ta[i] *= v;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Divides a constant to all elements in the array.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @param v the constant.\r\n\t */\r\n\tpublic static void divide(double[] a, double v) {\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ta[i] /= v;\r\n\t\t}\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the L2 norm of a vector.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @return the L2 norm of a vector.\r\n\t */\r\n\tpublic static double l2norm(double[] a) {\r\n\t\treturn Math.sqrt(StatUtils.sumSq(a));\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the L2 norm of a vector.\r\n\t * \r\n\t * @param v the vector.\r\n\t * @return the L2 norm of a vector.\r\n\t */\r\n\tpublic static double l2norm(Vector v) {\r\n\t\treturn l2norm(v.getValues());\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the L1 norm of a vector.\r\n\t * \r\n\t * @param a the vector.\r\n\t * @return the L1 norm of a vector.\r\n\t */\r\n\tpublic static double l1norm(double[] a) {\r\n\t\tdouble norm = 0;\r\n\t\tfor (double v : a) {\r\n\t\t\tnorm += Math.abs(v);\r\n\t\t}\r\n\t\treturn norm;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the L1 norm of a vector.\r\n\t * \r\n\t * @param v the vector.\r\n\t * @return the L1 norm of a vector.\r\n\t */\r\n\tpublic static double l1norm(Vector v) {\r\n\t\treturn l1norm(v.getValues());\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the dot product of two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the dot product of two vectors.\r\n\t */\r\n\tpublic static double dotProduct(double[] a, double[] b) {\r\n\t\tdouble s = 0;\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\ts += a[i] * b[i];\r\n\t\t}\r\n\t\treturn s;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the dot product of two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the dot product of two vectors.\r\n\t */\r\n\tpublic static double dotProduct(DenseVector a, DenseVector b) {\r\n\t\treturn dotProduct(a.getValues(), b.getValues());\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the dot product of two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the dot product of two vectors.\r\n\t */\r\n\tpublic static double dotProduct(SparseVector a, DenseVector b) {\r\n\t\tint[] indices1 = a.getIndices();\r\n\t\tdouble[] values1 = a.getValues();\r\n\t\tdouble[] values2 = b.getValues();\r\n\t\tdouble s = 0;\r\n\t\tfor (int i = 0; i < indices1.length; i++) {\r\n\t\t\ts += values1[i] * values2[indices1[i]];\r\n\t\t}\r\n\t\treturn s;\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the dot product of two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the dot product of two vectors.\r\n\t */\r\n\tpublic static double dotProduct(DenseVector a, SparseVector b) {\r\n\t\treturn dotProduct(b, a);\r\n\t}\r\n\r\n\t/**\r\n\t * Returns the dot product of two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the dot product of two vectors.\r\n\t */\r\n\tpublic static double dotProduct(SparseVector a, SparseVector b) {\r\n\t\tint[] indices1 = a.getIndices();\r\n\t\tdouble[] values1 = a.getValues();\r\n\t\tint[] indices2 = b.getIndices();\r\n\t\tdouble[] values2 = b.getValues();\r\n\t\tdouble s = 0;\r\n\t\tint i = 0;\r\n\t\tint j = 0;\r\n\t\twhile (i < indices1.length && j < indices2.length) {\r\n\t\t\tif (indices1[i] < indices2[j]) {\r\n\t\t\t\ti++;\r\n\t\t\t} else if (indices1[i] > indices2[j]) {\r\n\t\t\t\tj++;\r\n\t\t\t} else {\r\n\t\t\t\ts += values1[i] * values2[j];\r\n\t\t\t\ti++;\r\n\t\t\t\tj++;\r\n\t\t\t}\r\n\t\t}\r\n\t\treturn s;\r\n\t}\r\n\t\r\n\t/**\r\n\t * Returns the Pearson correlation coefficient between two vectors.\r\n\t * \r\n\t * @param a the 1st vector.\r\n\t * @param b the 2nd vector.\r\n\t * @return the Pearson correlation coefficient between two vectors.\r\n\t */\r\n\tpublic static double correlation(double[] a, double[] b) {\r\n\t\tdouble mean1 = StatUtils.mean(a);\r\n\t\tdouble mean2 = StatUtils.mean(b);\r\n\t\tdouble x = 0;\r\n\t\tdouble s1 = 0;\r\n\t\tdouble s2 = 0;\r\n\t\tfor (int i = 0; i < a.length; i++) {\r\n\t\t\tdouble d1 = (a[i] - mean1);\r\n\t\t\tdouble d2 = (b[i] - mean2);\r\n\t\t\tx += d1 * d2;\r\n\t\t\ts1 += d1 * d1;\r\n\t\t\ts2 += d2 * d2;\r\n\t\t}\r\n\t\treturn x / Math.sqrt(s1 * s2);\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/package-info.java",
    "content": "/**\n * Contains miscellaneous utility classes.\n */\npackage mltk.util;"
  },
  {
    "path": "src/main/java/mltk/util/tuple/DoublePair.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * CLass for $lt;double, double$gt; pair.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class DoublePair {\r\n\r\n\tpublic double v1;\r\n\tpublic double v2;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the 1st <code>double</code>.\r\n\t * @param v2 the 2nd <code>double</code>.\r\n\t */\r\n\tpublic DoublePair(double v1, double v2) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tlong temp;\r\n\t\ttemp = Double.doubleToLongBits(v1);\r\n\t\tresult = prime * result + (int) (temp ^ (temp >>> 32));\r\n\t\ttemp = Double.doubleToLongBits(v2);\r\n\t\tresult = prime * result + (int) (temp ^ (temp >>> 32));\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tDoublePair other = (DoublePair) obj;\r\n\t\tif (Double.doubleToLongBits(v1) != Double.doubleToLongBits(other.v1))\r\n\t\t\treturn false;\r\n\t\tif (Double.doubleToLongBits(v2) != Double.doubleToLongBits(other.v2))\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/IntDoublePair.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for &lt;int, double&gt; pairs.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class IntDoublePair {\r\n\r\n\tpublic int v1;\r\n\tpublic double v2;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the <code>int</code> value.\r\n\t * @param v2 the <code>double</code> value.\r\n\t */\r\n\tpublic IntDoublePair(int v1, double v2) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + v1;\r\n\t\tlong temp;\r\n\t\ttemp = Double.doubleToLongBits(v2);\r\n\t\tresult = prime * result + (int) (temp ^ (temp >>> 32));\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tIntDoublePair other = (IntDoublePair) obj;\r\n\t\tif (v1 != other.v1)\r\n\t\t\treturn false;\r\n\t\tif (Double.doubleToLongBits(v2) != Double.doubleToLongBits(other.v2))\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/IntDoublePairComparator.java",
    "content": "package mltk.util.tuple;\r\n\r\nimport java.util.Comparator;\r\n\r\n/**\r\n * Class for comparing &lt;int, double&gt; pairs. By default int is used as key, and in ascending order.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class IntDoublePairComparator implements Comparator<IntDoublePair> {\r\n\r\n\tprotected boolean ascending;\r\n\tprotected boolean firstIsKey;\r\n\r\n\tpublic IntDoublePairComparator() {\r\n\t\tthis(true, true);\r\n\t}\r\n\r\n\tpublic IntDoublePairComparator(boolean firstIsKey) {\r\n\t\tthis(firstIsKey, true);\r\n\t}\r\n\r\n\tpublic IntDoublePairComparator(boolean firstIsKey, boolean ascending) {\r\n\t\tthis.firstIsKey = firstIsKey;\r\n\t\tthis.ascending = ascending;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int compare(IntDoublePair o1, IntDoublePair o2) {\r\n\t\tint cmp = 0;\r\n\t\tif (firstIsKey) {\r\n\t\t\tcmp = Integer.compare(o1.v1, o2.v1);\r\n\t\t} else {\r\n\t\t\tcmp = Double.compare(o1.v2, o2.v2);\r\n\t\t}\r\n\t\tif (!ascending) {\r\n\t\t\tcmp = -cmp;\r\n\t\t}\r\n\t\treturn cmp;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/IntPair.java",
    "content": "package mltk.util.tuple;\n\n/**\n * Class for &lt;int, int&gt; pairs.\n * \n * @author Yin Lou\n * \n */\npublic class IntPair {\n\n\tpublic int v1;\n\tpublic int v2;\n\n\t/**\n\t * Constructor.\n\t * \n\t * @param v1 the 1st <code>int</code>.\n\t * @param v2 the 2nd <code>int</code>.\n\t */\n\tpublic IntPair(int v1, int v2) {\n\t\tthis.v1 = v1;\n\t\tthis.v2 = v2;\n\t}\n\n\t@Override\n\tpublic int hashCode() {\n\t\tfinal int prime = 31;\n\t\tint result = 1;\n\t\tresult = prime * result + v1;\n\t\tresult = prime * result + v2;\n\t\treturn result;\n\t}\n\n\t@Override\n\tpublic boolean equals(Object obj) {\n\t\tif (this == obj)\n\t\t\treturn true;\n\t\tif (obj == null)\n\t\t\treturn false;\n\t\tif (getClass() != obj.getClass())\n\t\t\treturn false;\n\t\tIntPair other = (IntPair) obj;\n\t\tif (v1 != other.v1)\n\t\t\treturn false;\n\t\tif (v2 != other.v2)\n\t\t\treturn false;\n\t\treturn true;\n\t}\n\n}\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/IntTriple.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for &lt;int, int, int&gt; triples.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class IntTriple {\r\n\r\n\tpublic int v1;\r\n\tpublic int v2;\r\n\tpublic int v3;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the 1st <code>int</code>.\r\n\t * @param v2 the 2nd <code>int</code>.\r\n\t * @param v3 the 3rd <code>int</code>.\r\n\t */\r\n\tpublic IntTriple(int v1, int v2, int v3) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t\tthis.v3 = v3;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + v1;\r\n\t\tresult = prime * result + v2;\r\n\t\tresult = prime * result + v3;\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tIntTriple other = (IntTriple) obj;\r\n\t\tif (v1 != other.v1)\r\n\t\t\treturn false;\r\n\t\tif (v2 != other.v2)\r\n\t\t\treturn false;\r\n\t\tif (v3 != other.v3)\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/LongDoublePair.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for &lt;long, double&gt; pairs.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class LongDoublePair {\r\n\r\n\tpublic long v1;\r\n\tpublic double v2;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the <code>long</code> value.\r\n\t * @param v2 the <code>double</code> value.\r\n\t */\r\n\tpublic LongDoublePair(long v1, double v2) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t}\r\n\t\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + (int) (v1 ^ (v1 >>> 32));\r\n\t\tlong temp;\r\n\t\ttemp = Double.doubleToLongBits(v2);\r\n\t\tresult = prime * result + (int) (temp ^ (temp >>> 32));\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tLongDoublePair other = (LongDoublePair) obj;\r\n\t\tif (v1 != other.v1)\r\n\t\t\treturn false;\r\n\t\tif (Double.doubleToLongBits(v2) != Double.doubleToLongBits(other.v2))\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic String toString() {\r\n\t\treturn \"(\" + v1 + \", \" + v2 + \")\";\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/LongDoublePairComparator.java",
    "content": "package mltk.util.tuple;\r\n\r\nimport java.util.Comparator;\r\n\r\n/**\r\n * Class for comparing &lt;long, double&gt; pairs. By default long is used as key, and in ascending order.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class LongDoublePairComparator implements Comparator<LongDoublePair> {\r\n\r\n\tprotected boolean ascending;\r\n\tprotected boolean firstIsKey;\r\n\r\n\tpublic LongDoublePairComparator() {\r\n\t\tthis(true, true);\r\n\t}\r\n\r\n\tpublic LongDoublePairComparator(boolean firstIsKey) {\r\n\t\tthis(firstIsKey, true);\r\n\t}\r\n\r\n\tpublic LongDoublePairComparator(boolean firstIsKey, boolean ascending) {\r\n\t\tthis.firstIsKey = firstIsKey;\r\n\t\tthis.ascending = ascending;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int compare(LongDoublePair o1, LongDoublePair o2) {\r\n\t\tint cmp = 0;\r\n\t\tif (firstIsKey) {\r\n\t\t\tcmp = Long.compare(o1.v1, o2.v1);\r\n\t\t} else {\r\n\t\t\tcmp = Double.compare(o1.v2, o2.v2);\r\n\t\t}\r\n\t\tif (!ascending) {\r\n\t\t\tcmp = -cmp;\r\n\t\t}\r\n\t\treturn cmp;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/LongPair.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for &lt;long, long&gt; pairs.\r\n * \r\n * @author Yin Lou\r\n * \r\n */\r\npublic class LongPair {\r\n\r\n\tpublic long v1;\r\n\tpublic long v2;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the 1st <code>long</code>.\r\n\t * @param v2 the 2nd <code>long</code>.\r\n\t */\r\n\tpublic LongPair(long v1, long v2) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + (int) (v1 ^ (v1 >>> 32));\r\n\t\tresult = prime * result + (int) (v2 ^ (v2 >>> 32));\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tLongPair other = (LongPair) obj;\r\n\t\tif (v1 != other.v1)\r\n\t\t\treturn false;\r\n\t\tif (v2 != other.v2)\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/Pair.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for generic pairs.\r\n * \r\n * @author Yin Lou\r\n * \r\n * @param <T1> the type of the 1st element.\r\n * @param <T2> the type of the 2nd element.\r\n */\r\npublic class Pair<T1, T2> {\r\n\r\n\tpublic T1 v1;\r\n\tpublic T2 v2;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the 1st element.\r\n\t * @param v2 the 2nd element.\r\n\t */\r\n\tpublic Pair(T1 v1, T2 v2) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + ((v1 == null) ? 0 : v1.hashCode());\r\n\t\tresult = prime * result + ((v2 == null) ? 0 : v2.hashCode());\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tPair<?, ?> other = (Pair<?, ?>) obj;\r\n\t\tif (v1 == null) {\r\n\t\t\tif (other.v1 != null)\r\n\t\t\t\treturn false;\r\n\t\t} else if (v1.getClass() != other.v1.getClass()) {\r\n\t\t\treturn false;\r\n\t\t} else if (!v1.equals(other.v1)) {\r\n\t\t\treturn false;\r\n\t\t}\r\n\t\tif (v2 == null) {\r\n\t\t\tif (other.v2 != null)\r\n\t\t\t\treturn false;\r\n\t\t} else if (v2.getClass() != other.v2.getClass()) {\r\n\t\t\treturn false;\r\n\t\t} else if (!v2.equals(other.v2)) {\r\n\t\t\treturn false;\r\n\t\t}\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/Triple.java",
    "content": "package mltk.util.tuple;\r\n\r\n/**\r\n * Class for generic triples.\r\n * \r\n * @author Yin Lou\r\n * \r\n * @param <T1> the type of the 1st element.\r\n * @param <T2> the type of the 2nd element.\r\n * @param <T3> the type of the 3rd element.\r\n */\r\npublic class Triple<T1, T2, T3> {\r\n\r\n\tpublic T1 v1;\r\n\tpublic T2 v2;\r\n\tpublic T3 v3;\r\n\r\n\t/**\r\n\t * Constructor.\r\n\t * \r\n\t * @param v1 the 1st element.\r\n\t * @param v2 the 2nd element.\r\n\t * @param v3 the 3rd element.\r\n\t */\r\n\tpublic Triple(T1 v1, T2 v2, T3 v3) {\r\n\t\tthis.v1 = v1;\r\n\t\tthis.v2 = v2;\r\n\t\tthis.v3 = v3;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic int hashCode() {\r\n\t\tfinal int prime = 31;\r\n\t\tint result = 1;\r\n\t\tresult = prime * result + ((v1 == null) ? 0 : v1.hashCode());\r\n\t\tresult = prime * result + ((v2 == null) ? 0 : v2.hashCode());\r\n\t\tresult = prime * result + ((v3 == null) ? 0 : v3.hashCode());\r\n\t\treturn result;\r\n\t}\r\n\r\n\t@Override\r\n\tpublic boolean equals(Object obj) {\r\n\t\tif (this == obj)\r\n\t\t\treturn true;\r\n\t\tif (obj == null)\r\n\t\t\treturn false;\r\n\t\tif (getClass() != obj.getClass())\r\n\t\t\treturn false;\r\n\t\tTriple<?, ?, ?> other = (Triple<?, ?, ?>) obj;\r\n\t\tif (v1 == null) {\r\n\t\t\tif (other.v1 != null)\r\n\t\t\t\treturn false;\r\n\t\t} else if (!v1.equals(other.v1))\r\n\t\t\treturn false;\r\n\t\tif (v2 == null) {\r\n\t\t\tif (other.v2 != null)\r\n\t\t\t\treturn false;\r\n\t\t} else if (!v2.equals(other.v2))\r\n\t\t\treturn false;\r\n\t\tif (v3 == null) {\r\n\t\t\tif (other.v3 != null)\r\n\t\t\t\treturn false;\r\n\t\t} else if (!v3.equals(other.v3))\r\n\t\t\treturn false;\r\n\t\treturn true;\r\n\t}\r\n\r\n}\r\n"
  },
  {
    "path": "src/main/java/mltk/util/tuple/package-info.java",
    "content": "/**\n * Contains utility classes for tuples.\n */\npackage mltk.util.tuple;"
  },
  {
    "path": "src/test/java/mltk/core/BinsTest.java",
    "content": "package mltk.core;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.util.MathUtils;\n\npublic class BinsTest {\n\n\t@Test\n\tpublic void testBins() {\n\t\tBins bins = new Bins(new double[] {1, 5, 6}, new double[] {0.5, 2.5, 5.5});\n\t\t\n\t\tAssert.assertEquals(0, bins.getIndex(-1));\n\t\tAssert.assertEquals(0, bins.getIndex(0.3));\n\t\tAssert.assertEquals(0, bins.getIndex(1));\n\t\tAssert.assertEquals(1, bins.getIndex(1.1));\n\t\tAssert.assertEquals(1, bins.getIndex(5));\n\t\tAssert.assertEquals(2, bins.getIndex(5.5));\n\t\tAssert.assertEquals(2, bins.getIndex(6.5));\n\t\t\n\t\tAssert.assertEquals(0.5, bins.getValue(0), MathUtils.EPSILON);\n\t\tAssert.assertEquals(2.5, bins.getValue(1), MathUtils.EPSILON);\n\t\tAssert.assertEquals(5.5, bins.getValue(2), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/core/InstancesTestHelper.java",
    "content": "package mltk.core;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class InstancesTestHelper {\n\t\n\tprivate static InstancesTestHelper instance = null;\n\t\n\tprivate Instances denseClaDataset;\n\tprivate Instances denseRegDataset;\n\tprivate Instances denseClaDatasetWMissing;\n\tprivate Instances denseRegDatasetWMissing;\n\t\n\tpublic static InstancesTestHelper getInstance() {\n\t\tif (instance == null) {\n\t\t\tinstance = new InstancesTestHelper();\n\t\t}\n\t\treturn instance;\n\t}\n\t\n\tpublic Instances getDenseClassificationDataset() {\n\t\treturn denseClaDataset;\n\t}\n\t\n\tpublic Instances getDenseRegressionDataset() {\n\t\treturn denseRegDataset;\n\t}\n\t\n\tpublic Instances getDenseClassificationDatasetWMissing() {\n\t\treturn denseClaDatasetWMissing;\n\t}\n\t\n\tpublic Instances getDenseRegressionDatasetWMissing() {\n\t\treturn denseRegDatasetWMissing;\n\t}\n\t\n\tprivate InstancesTestHelper() {\n\t\tList<Attribute> attributes = new ArrayList<>();\n\t\t\n\t\tNumericalAttribute f1 = new NumericalAttribute(\"f1\", 0);\n\t\tNominalAttribute f2 = new NominalAttribute(\"f2\", new String[] {\"a\", \"b\", \"c\"}, 1);\n\t\tBinnedAttribute f3 = new BinnedAttribute(\"f3\", 256, 2);\n\t\tBins bins = new Bins(new double[] {1, 5, 6}, new double[] {0.5, 2.5, 3});\n\t\tBinnedAttribute f4 = new BinnedAttribute(\"f4\", bins, 3);\n\t\t\n\t\tattributes.add(f1);\n\t\tattributes.add(f2);\n\t\tattributes.add(f3);\n\t\tattributes.add(f4);\n\t\t\n\t\tAttribute claTarget = new NominalAttribute(\"target\", new String[] {\"0\", \"1\"});\n\t\tAttribute regTarget = new NumericalAttribute(\"target\");\n\t\t\n\t\tdenseClaDataset = new Instances(attributes, claTarget);\n\t\tfor (int i = 0; i < 1000; i++) {\n\t\t\tdouble[] v = new double[4];\n\t\t\tv[0] = i * 0.1;\n\t\t\tv[1] = i % f2.getCardinality();\n\t\t\tv[2] = (i + 1000) % f3.getNumBins();\n\t\t\tv[3] = i % f3.getNumBins();\n\t\t\tdouble target = (i % 10) < 8 ? 0 : 1;\n\t\t\tInstance instance = new Instance(v, target);\n\t\t\tdenseClaDataset.add(instance);\n\t\t}\n\t\t\n\t\tdenseRegDataset = denseClaDataset.copy();\n\t\tdenseRegDataset.setTargetAttribute(regTarget);\n\t\t\n\t\tdenseClaDatasetWMissing = denseClaDataset.copy();\n\t\tfor (int i = 0; i < 10; i++) {\n\t\t\tdenseClaDatasetWMissing.get(i).setValue(0, Double.NaN);\n\t\t}\n\t\tdenseRegDatasetWMissing = denseClaDatasetWMissing.copy();\n\t\tdenseRegDatasetWMissing.setTargetAttribute(regTarget);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/core/io/AttributesReaderTest.java",
    "content": "package mltk.core.io;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\nimport java.util.List;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Attribute;\nimport mltk.core.Attribute.Type;\nimport mltk.core.BinnedAttribute;\nimport mltk.core.NominalAttribute;\nimport mltk.util.tuple.Pair;\n\npublic class AttributesReaderTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\tPrintWriter out = new PrintWriter(boas);\n\t\tout.println(\"f1: cont\");\n\t\tout.println(\"f2: {a, b, c}\");\n\t\tout.println(\"f3: binned (256)\");\n\t\tout.println(\"f4: binned (3;[1, 5, 6];[0.5, 2.5, 3])\");\n\t\tout.println(\"label: cont (target)\");\n\t\tout.flush();\n\t\tout.close();\n\t\t\n\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\tBufferedReader br = new BufferedReader(new InputStreamReader(bais));\n\t\tPair<List<Attribute>, Attribute> pair = null;\n\t\ttry {\n\t\t\tpair = AttributesReader.read(br);\n\t\t} catch (IOException e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t\t\n\t\tList<Attribute> attributes = pair.v1;\n\t\tAttribute targetAtt = pair.v2;\n\t\tAssert.assertEquals(\"label\", targetAtt.getName());\n\t\tAssert.assertEquals(4, attributes.size());\n\t\tfor (int i = 0; i < attributes.size(); i++) {\n\t\t\tAssert.assertEquals(i, attributes.get(i).getIndex());\n\t\t}\n\t\tAssert.assertEquals(Type.NUMERIC, attributes.get(0).getType());\n\t\tAssert.assertEquals(Type.NOMINAL, attributes.get(1).getType());\n\t\tAssert.assertEquals(Type.BINNED, attributes.get(2).getType());\n\t\tAssert.assertEquals(Type.BINNED, attributes.get(3).getType());\n\t\tAssert.assertArrayEquals(new String[] {\"a\", \"b\", \"c\"},\n\t\t\t\t((NominalAttribute) attributes.get(1)).getStates());\n\t\tAssert.assertEquals(256, ((BinnedAttribute) attributes.get(2)).getNumBins());\n\t\tAssert.assertEquals(3, ((BinnedAttribute) attributes.get(3)).getNumBins());\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/core/io/InstancesReaderTest.java",
    "content": "package mltk.core.io;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Instance;\n\npublic class InstancesReaderTest {\n\n\t@Test\n\tpublic void testDenseFormat() {\n\t\tString[] data = {\"0.0\", \"1.5\", \"?\", \"3\"};\n\t\tInstance instance = InstancesReader.parseDenseInstance(data, 3);\n\t\tAssert.assertTrue(instance.isMissing(2));\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/core/processor/DiscretizerTest.java",
    "content": "package mltk.core.processor;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.BinnedAttribute;\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\nimport mltk.util.MathUtils;\n\npublic class DiscretizerTest {\n\t\n\t@Test\n\tpublic void testMissingValue() {\n\t\tInstances instances = InstancesTestHelper.getInstance().getDenseClassificationDatasetWMissing().copy();\n\t\tDiscretizer.discretize(instances, 0, 10);\n\t\tAssert.assertTrue(instances.getAttributes().get(0).getClass() == BinnedAttribute.class);\n\t\tfor (int i = 0; i < 10; i++) {\n\t\t\tAssert.assertTrue(instances.get(i).isMissing(0));\n\t\t}\n\t\tfor (int i = 10; i < 20; i++) {\n\t\t\tAssert.assertFalse(instances.get(i).isMissing(0));\n\t\t\tAssert.assertTrue(MathUtils.isInteger(instances.get(i).getValue(0)));\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/core/processor/InstancesSplitterTest.java",
    "content": "package mltk.core.processor;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\nimport mltk.util.Random;\n\npublic class InstancesSplitterTest {\n\t\n\t@Test\n\tpublic void testSamplingTrainValid() {\n\t\tRandom.getInstance().setSeed(5);\n\t\tInstances instances = InstancesTestHelper.getInstance()\n\t\t\t\t.getDenseRegressionDataset();\n\t\tInstances[] datasets = InstancesSplitter.split(instances, 0.8);\n\t\tInstances train = datasets[0];\n\t\tInstances valid = datasets[1];\n\t\tAssert.assertEquals((int) (instances.size() * 0.8), train.size());\n\t\tAssert.assertEquals((int) (instances.size() * 0.2), valid.size());\n\t}\n\t\n\t@Test\n\tpublic void testSamplingTrainValidTest() {\n\t\tRandom.getInstance().setSeed(5);\n\t\tInstances instances = InstancesTestHelper.getInstance()\n\t\t\t\t.getDenseRegressionDataset();\n\t\tInstances[] datasets = InstancesSplitter.split(instances, 0.7, 0.1, 0.1);\n\t\tInstances train = datasets[0];\n\t\tInstances valid = datasets[1];\n\t\tInstances test = datasets[2];\n\t\tAssert.assertEquals((int) (instances.size() * 0.7), train.size());\n\t\tAssert.assertEquals((int) (instances.size() * 0.1), valid.size());\n\t\tAssert.assertEquals((int) (instances.size() * 0.1), test.size());\n\t}\n\t\n\t@Test\n\tpublic void testStratifiedSampling() {\n\t\tRandom.getInstance().setSeed(5);\n\t\tInstances instances = InstancesTestHelper.getInstance()\n\t\t\t\t.getDenseClassificationDataset();\n\t\tInstances[] datasets = InstancesSplitter.split(instances, \"target\", 0.8, 0.2);\n\t\tInstances train = datasets[0];\n\t\tInstances valid = datasets[1];\n\t\tint numPosTrain = 0;\n\t\tfor (Instance instance : train) {\n\t\t\tif (instance.getTarget() == 1) {\n\t\t\t\tnumPosTrain++;\n\t\t\t}\n\t\t}\n\t\tint numPosValid = 0;\n\t\tfor (Instance instance : valid) {\n\t\t\tif (instance.getTarget() == 1) {\n\t\t\t\tnumPosValid++;\n\t\t\t}\n\t\t}\n\t\tAssert.assertTrue(numPosTrain < train.size() / 3);\n\t\tAssert.assertTrue(numPosValid < valid.size() / 3);\n\t\tAssert.assertEquals((int) (instances.size() * 0.8), train.size());\n\t\tAssert.assertEquals((int) (instances.size() * 0.2), valid.size());\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/AUCTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class AUCTest {\n\t\n\t@Test\n\tpublic void test1() {\n\t\tdouble[] preds = {0.8, 0.1, 0.05, 0.9};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tAUC metric = new AUC();\n\t\tAssert.assertEquals(1, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void test2() {\n\t\tdouble[] preds = {0.5, 0.5, 0.5, 0.5};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tAUC metric = new AUC();\n\t\tAssert.assertEquals(0.5, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/ConvergenceTesterTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ConvergenceTesterTest {\n\n\t@Test\n\tpublic void test1() {\n\t\tConvergenceTester ct = new ConvergenceTester(10, 0, 0.8);\n\t\tct.setMetric(new AUC());\n\t\tct.add(0.55);\n\t\tct.add(0.60);\n\t\tct.add(0.70);\n\t\tct.add(0.75);\n\t\tct.add(0.80);\n\t\tct.add(0.85);\n\t\tct.add(0.90); // Peak\n\t\tct.add(0.85);\n\t\tAssert.assertFalse(ct.isConverged());\n\t\tct.add(0.82);\n\t\tct.add(0.81);\n\t\tAssert.assertTrue(ct.isConverged());\n\t}\n\t\n\t@Test\n\tpublic void test2() {\n\t\tConvergenceTester ct = new ConvergenceTester(10, 2, 1.0);\n\t\tct.setMetric(new AUC());\n\t\tct.add(0.55);\n\t\tct.add(0.60);\n\t\tct.add(0.70);\n\t\tct.add(0.75);\n\t\tct.add(0.80);\n\t\tct.add(0.85);\n\t\tct.add(0.90); // Peak\n\t\tct.add(0.85);\n\t\tAssert.assertFalse(ct.isConverged());\n\t\tct.add(0.82);\n\t\tct.add(0.81);\n\t\tAssert.assertTrue(ct.isConverged());\n\t}\n\t\n\t@Test\n\tpublic void test3() {\n\t\tConvergenceTester ct = new ConvergenceTester(10, 0, 0.8);\n\t\tct.setMetric(new RMSE());\n\t\tct.add(5.00);\n\t\tct.add(4.50);\n\t\tct.add(4.00);\n\t\tct.add(3.50);\n\t\tct.add(3.00);\n\t\tct.add(2.50);\n\t\tct.add(2.00); // Bottom\n\t\tct.add(2.20);\n\t\tAssert.assertFalse(ct.isConverged());\n\t\tct.add(2.10);\n\t\tct.add(2.05);\n\t\tAssert.assertTrue(ct.isConverged());\n\t}\n\t\n\t@Test\n\tpublic void test4() {\n\t\tConvergenceTester ct = new ConvergenceTester(10, 2, 1.0);\n\t\tct.setMetric(new RMSE());\n\t\tct.add(5.00);\n\t\tct.add(4.50);\n\t\tct.add(4.00);\n\t\tct.add(3.50);\n\t\tct.add(3.00);\n\t\tct.add(2.50);\n\t\tct.add(2.00); // Bottom\n\t\tct.add(2.20);\n\t\tAssert.assertFalse(ct.isConverged());\n\t\tct.add(2.10);\n\t\tct.add(2.05);\n\t\tAssert.assertTrue(ct.isConverged());\n\t}\n\t\n\t@Test\n\tpublic void testParse() {\n\t\tConvergenceTester ct = null;\n\n\t    // Empty\n\t    ct = ConvergenceTester.parse(\"\");\n\t    Assert.assertEquals(ct.minNumPoints, -1);\n\t    Assert.assertEquals(ct.n, 0);\n\t    Assert.assertEquals(ct.c, 1.0, 1e-6);\n\n\t    // One parameter\n\t    ct = ConvergenceTester.parse(\"10\");\n\t    Assert.assertEquals(ct.minNumPoints, 10);\n\t    Assert.assertEquals(ct.n, 0);\n\t    Assert.assertEquals(ct.c, 1.0, 1e-6);\n\t    \n\t    // Two parameters\n\t    ct = ConvergenceTester.parse(\"10:5\");\n\t    Assert.assertEquals(ct.minNumPoints, 10);\n\t    Assert.assertEquals(ct.n, 5);\n\t    Assert.assertEquals(ct.c, 1.0, 1e-6);\n\t    \n\n\t    // Three parameters\n\t    ct = ConvergenceTester.parse(\"10:5:0.8\");\n\t    Assert.assertEquals(ct.minNumPoints, 10);\n\t    Assert.assertEquals(ct.n, 5);\n\t    Assert.assertEquals(ct.c, 0.8, 1e-6);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/ErrorTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ErrorTest {\n\t\n\t@Test\n\tpublic void testLabel() {\n\t\tdouble[] preds = {1, 0, 1, 0};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tError metric = new Error();\n\t\tAssert.assertEquals(0.5, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\n\t@Test\n\tpublic void testProbability() {\n\t\tdouble[] preds = {2, -1.5, 0.3, 5};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tError metric = new Error();\n\t\tAssert.assertEquals(0.25, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/LogLossTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class LogLossTest {\n\t\n\t@Test\n\tpublic void testProb() {\n\t\tdouble[] preds = {0.8, 0.1, 0.05, 0.9};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tLogLoss metric = new LogLoss(false);\n\t\tAssert.assertEquals(0.485157877, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testRawScore() {\n\t\tdouble[] preds = {5, -5, -3, 3};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tLogLoss metric = new LogLoss(true);\n\t\tAssert.assertEquals(0.1106054, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/LogisticLossTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class LogisticLossTest {\n\t\n\t@Test\n\tpublic void test() {\n\t\tdouble[] preds = {5, -5, -3, 3};\n\t\tdouble[] targets = {1, 0, 0, 1};\n\t\tLogisticLoss metric = new LogisticLoss();\n\t\tAssert.assertEquals(0.02765135, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/MAETest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class MAETest {\n\n\t@Test\n\tpublic void test() {\n\t\tdouble[] preds = {1, 2, 3, 4};\n\t\tdouble[] targets = {0.1, 0.2, 0.3, 0.4};\n\t\tMAE metric = new MAE();\n\t\tAssert.assertEquals(2.25, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/MetricFactoryTest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class MetricFactoryTest {\n\t\n\t@Test\n\tpublic void test() {\n\t\tAssert.assertEquals(AUC.class, MetricFactory.getMetric(\"AUC\").getClass());\n\t\tAssert.assertEquals(Error.class, MetricFactory.getMetric(\"Error\").getClass());\n\t\tAssert.assertEquals(LogisticLoss.class, MetricFactory.getMetric(\"LogisticLoss\").getClass());\n\t\tAssert.assertEquals(LogLoss.class, MetricFactory.getMetric(\"LogLoss\").getClass());\n\t\tAssert.assertEquals(LogLoss.class, MetricFactory.getMetric(\"LogLoss:True\").getClass());\n\t\tAssert.assertEquals(true, ((LogLoss) MetricFactory.getMetric(\"LogLoss:True\")).isRawScore());\n\t\tAssert.assertEquals(MAE.class, MetricFactory.getMetric(\"MAE\").getClass());\n\t\tAssert.assertEquals(RMSE.class, MetricFactory.getMetric(\"RMSE\").getClass());\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/evaluation/RMSETest.java",
    "content": "package mltk.predictor.evaluation;\n\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class RMSETest {\n\n\t@Test\n\tpublic void test() {\n\t\tdouble[] preds = {1, 2, 3, 4};\n\t\tdouble[] targets = {1.1, 1.9, 3.2, 4};\n\t\tRMSE metric = new RMSE();\n\t\tAssert.assertEquals(0.122474487, metric.eval(preds, targets), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/glm/GLMTest.java",
    "content": "package mltk.predictor.glm;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.MathUtils;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class GLMTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tdouble[] intercept = {1.0, -1.0};\n\t\tdouble[][] w = {\n\t\t\t\t{1, 2, 3},\n\t\t\t\t{-1, -2, -3}\n\t\t};\n\t\tGLM glm = new GLM(intercept, w);\n\t\t\n\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\tPrintWriter out = new PrintWriter(boas);\n\t\ttry {\n\t\t\tglm.write(out);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t\tout.flush();\n\t\tout.close();\n\t\t\n\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\tBufferedReader br = new BufferedReader(new InputStreamReader(bais));\n\t\ttry {\n\t\t\tGLM parsedGLM = PredictorReader.read(br, GLM.class);\n\t\t\tAssert.assertEquals(intercept.length, parsedGLM.intercept().length);\n\t\t\tAssert.assertEquals(w.length, parsedGLM.coefficients().length);\n\t\t\tAssert.assertArrayEquals(intercept, parsedGLM.intercept, MathUtils.EPSILON);\n\t\t\tfor (int i = 0; i < intercept.length; i++) {\n\t\t\t\tAssert.assertArrayEquals(w[i], parsedGLM.coefficients(i), MathUtils.EPSILON);\n\t\t\t}\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/DecisionTableLearnerTest.java",
    "content": "package mltk.predictor.tree;\n\nimport java.util.List;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\n\npublic class DecisionTableLearnerTest {\n\n\t@Test\n\tpublic void testDecisionTableLearner1() {\n\t\tDecisionTableLearner rtLearner = new DecisionTableLearner();\n\t\trtLearner.setConstructionMode(DecisionTableLearner.Mode.ONE_PASS_GREEDY);\n\t\trtLearner.setMaxDepth(2);\n\t\t\n\t\tInstances instances = InstancesTestHelper.getInstance().getDenseRegressionDataset();\n\t\tDecisionTable rt = rtLearner.build(instances);\n\t\tint[] attributeIndices = rt.getAttributeIndices();\n\t\tAssert.assertEquals(2, attributeIndices.length);\n\t\tAssert.assertEquals(0, attributeIndices[0]);\n\t}\n\t\n\t@Test\n\tpublic void testDecisionTableLearner2() {\n\t\tDecisionTableLearner rtLearner = new DecisionTableLearner();\n\t\trtLearner.setConstructionMode(DecisionTableLearner.Mode.ONE_PASS_GREEDY);\n\t\trtLearner.setMaxDepth(2);\n\t\t\n\t\tInstances instances = InstancesTestHelper.getInstance()\n\t\t\t\t.getDenseRegressionDataset().copy();\n\t\t// Apply feature selection\n\t\tList<Attribute> attributes = instances.getAttributes(1);\n\t\tinstances.setAttributes(attributes);\n\t\tDecisionTable rt = rtLearner.build(instances);\n\t\tint[] attributeIndices = rt.getAttributeIndices();\n\t\tAssert.assertEquals(2, attributeIndices.length);\n\t\tAssert.assertEquals(1, attributeIndices[0]);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/DecisionTableTest.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Instance;\nimport mltk.core.InstancesTestHelper;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.MathUtils;\n\npublic class DecisionTableTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tDecisionTable dt = DecisionTableTestHelper.getInstance().getTable1();\n\t\tInstance instance = InstancesTestHelper.getInstance().getDenseRegressionDataset().get(0);\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\tdt.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\tDecisionTable t = PredictorReader.read(in, DecisionTable.class);\n\t\t\t\n\t\t\tAssert.assertArrayEquals(dt.getAttributeIndices(), t.getAttributeIndices());\n\t\t\tAssert.assertArrayEquals(dt.getSplits(), t.getSplits(), MathUtils.EPSILON);\n\t\t\tAssert.assertEquals(dt.regress(instance), t.regress(instance), MathUtils.EPSILON);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\t\n\t@Test\n\tpublic void testRegress() {\n\t\tDecisionTable dt = DecisionTableTestHelper.getInstance().getTable1();\n\t\tInstance instance = InstancesTestHelper.getInstance().getDenseRegressionDataset().get(0);\n\t\t\n\t\tAssert.assertEquals(0.7, dt.regress(instance), MathUtils.EPSILON);\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/DecisionTableTestHelper.java",
    "content": "package mltk.predictor.tree;\n\npublic class DecisionTableTestHelper {\n\t\n\tprivate static DecisionTableTestHelper instance = null;\n\t\n\tprivate DecisionTable dt1;\n\tprivate DecisionTable dt2;\n\t\n\tpublic static DecisionTableTestHelper getInstance() {\n\t\tif (instance == null) {\n\t\t\tinstance = new DecisionTableTestHelper();\n\t\t}\n\t\treturn instance;\n\t}\n\t\n\tpublic DecisionTable getTable1() {\n\t\treturn dt1;\n\t}\n\t\n\tpublic DecisionTable getTable2() {\n\t\treturn dt2;\n\t}\n\t\n\tprivate DecisionTableTestHelper() {\n\t\tbuildDecisionTable1();\n\t\tbuildDecisionTable2();\n\t}\n\t\n\tprivate void buildDecisionTable1() {\n\t\tint[] attIndices = new int[] {0, 1, 2};\n\t\tdouble[] splits = new double[] {50, 1.5, 23.5};\n\t\tlong[] predIndices = new long[] {0, 1, 2, 3, 4, 5, 6, 7};\n\t\tdouble[] predValues = new double[] {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8};\n\t\t\n\t\tdt1 = new DecisionTable(attIndices, splits, predIndices, predValues);\n\t}\n\t\n\tprivate void buildDecisionTable2() {\n\t\tint[] attIndices = new int[] {3, 2, 0};\n\t\tdouble[] splits = new double[] {1, 56.5, 20};\n\t\tlong[] predIndices = new long[] {0, 1, 2, 3, 4, 5, 6, 7};\n\t\tdouble[] predValues = new double[] {1.0, -0.9, 0.8, -0.7, 0.6, -0.5, 0.4, -0.3, 0.2};\n\t\t\n\t\tdt2 = new DecisionTable(attIndices, splits, predIndices, predValues);\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/RegressionTreeLearnerTest.java",
    "content": "package mltk.predictor.tree;\n\nimport java.util.List;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Attribute;\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\n\npublic class RegressionTreeLearnerTest {\n\n\t@Test\n\tpublic void testRegressionTreeLearner1() {\n\t\tRegressionTreeLearner rtLearner = new RegressionTreeLearner();\n\t\trtLearner.setConstructionMode(RegressionTreeLearner.Mode.DEPTH_LIMITED);\n\t\trtLearner.setMaxDepth(2);\n\t\t\n\t\tInstances instances = InstancesTestHelper.getInstance().getDenseRegressionDataset();\n\t\tRegressionTree rt = rtLearner.build(instances);\n\t\tTreeInteriorNode root = (TreeInteriorNode) rt.getRoot();\n\t\tAssert.assertEquals(0, root.attIndex);\n\t\tAssert.assertTrue(root.getLeftChild() != null);\n\t\tAssert.assertTrue(root.getRightChild() != null);\n\t}\n\t\n\t@Test\n\tpublic void testRegressionTreeLearner2() {\n\t\tRegressionTreeLearner rtLearner = new RegressionTreeLearner();\n\t\trtLearner.setConstructionMode(RegressionTreeLearner.Mode.DEPTH_LIMITED);\n\t\trtLearner.setMaxDepth(2);\n\t\t\n\t\tInstances instances = InstancesTestHelper.getInstance()\n\t\t\t\t.getDenseRegressionDataset().copy();\n\t\t// Apply feature selection\n\t\tList<Attribute> attributes = instances.getAttributes(1);\n\t\tinstances.setAttributes(attributes);\n\t\tRegressionTree rt = rtLearner.build(instances);\n\t\tTreeInteriorNode root = (TreeInteriorNode) rt.getRoot();\n\t\tAssert.assertEquals(1, root.attIndex);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/RegressionTreeTest.java",
    "content": "package mltk.predictor.tree;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.predictor.io.PredictorReader;\nimport mltk.util.MathUtils;\n\npublic class RegressionTreeTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tRegressionTree tree = RegressionTreeTestHelper.getInstance().getTree1();\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\ttree.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\tRegressionTree t = PredictorReader.read(in, RegressionTree.class);\n\t\t\t\n\t\t\tAssert.assertTrue(t.root instanceof TreeInteriorNode);\n\t\t\tTreeInteriorNode root = (TreeInteriorNode) t.root;\n\t\t\tAssert.assertEquals(1, root.getSplitAttributeIndex());\n\t\t\tAssert.assertEquals(0.5, root.getSplitPoint(), MathUtils.EPSILON);\n\t\t\tAssert.assertTrue(root.left instanceof RegressionTreeLeaf);\n\t\t\tAssert.assertTrue(root.right instanceof TreeInteriorNode);\n\t\t\t\n\t\t\tRegressionTreeLeaf leaf1 = (RegressionTreeLeaf) root.left;\n\t\t\tAssert.assertEquals(0.4, leaf1.getPrediction(), MathUtils.EPSILON);\n\t\t\t\n\t\t\tTreeInteriorNode right = (TreeInteriorNode) root.right;\n\t\t\tAssert.assertEquals(2, right.getSplitAttributeIndex());\n\t\t\tAssert.assertEquals(-1.5, right.getSplitPoint(), MathUtils.EPSILON);\n\t\t\tAssert.assertTrue(right.left.isLeaf());\n\t\t\tAssert.assertTrue(right.right.isLeaf());\n\t\t\t\n\t\t\tRegressionTreeLeaf leaf2 = (RegressionTreeLeaf) right.left;\n\t\t\tAssert.assertEquals(0.5, leaf2.getPrediction(), MathUtils.EPSILON);\n\t\t\t\n\t\t\tRegressionTreeLeaf leaf3 = (RegressionTreeLeaf) right.right;\n\t\t\tAssert.assertEquals(0.6, leaf3.getPrediction(), MathUtils.EPSILON);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/RegressionTreeTestHelper.java",
    "content": "package mltk.predictor.tree;\n\npublic class RegressionTreeTestHelper {\n\t\n\tprivate static RegressionTreeTestHelper instance = null;\n\t\n\tprivate RegressionTree tree1;\n\tprivate RegressionTree tree2;\n\t\n\tpublic static RegressionTreeTestHelper getInstance() {\n\t\tif (instance == null) {\n\t\t\tinstance = new RegressionTreeTestHelper();\n\t\t}\n\t\treturn instance;\n\t}\n\t\n\tpublic RegressionTree getTree1() {\n\t\treturn tree1;\n\t}\n\t\n\tpublic RegressionTree getTree2() {\n\t\treturn tree2;\n\t}\n\t\n\tprivate RegressionTreeTestHelper() {\n\t\tbuildTree1();\n\t\tbuildTree2();\n\t}\n\t\n\tprivate void buildTree1() {\n\t\tTreeInteriorNode root = new TreeInteriorNode(1, 0.5);\n\t\tTreeNode leaf1 = new RegressionTreeLeaf(0.4);\n\t\tTreeInteriorNode right = new TreeInteriorNode(2, -1.5);\n\t\tTreeNode leaf2 = new RegressionTreeLeaf(0.5);\n\t\tTreeNode leaf3 = new RegressionTreeLeaf(0.6);\n\t\tright.left = leaf2;\n\t\tright.right = leaf3;\n\t\troot.left = leaf1;\n\t\troot.right = right;\n\t\t\n\t\ttree1 = new RegressionTree(root);\n\t}\n\t\n\tprivate void buildTree2() {\n\t\tTreeInteriorNode root = new TreeInteriorNode(5, 0);\n\t\tTreeNode leaf1 = new RegressionTreeLeaf(-0.4);\n\t\tTreeInteriorNode left = new TreeInteriorNode(0, -3.5);\n\t\tTreeNode leaf2 = new RegressionTreeLeaf(-0.5);\n\t\tTreeNode leaf3 = new RegressionTreeLeaf(-0.6);\n\t\tleft.left = leaf1;\n\t\tleft.right = leaf2;\n\t\troot.left = left;\n\t\troot.right = leaf3;\n\t\t\n\t\ttree2 = new RegressionTree(root);\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/BoostedDTablesTest.java",
    "content": "package mltk.predictor.tree.ensemble;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.predictor.tree.DecisionTable;\nimport mltk.predictor.tree.DecisionTableTestHelper;\nimport mltk.util.MathUtils;\n\npublic class BoostedDTablesTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tDecisionTable dt1 = DecisionTableTestHelper.getInstance().getTable1();\n\t\tDecisionTable dt2 = DecisionTableTestHelper.getInstance().getTable2();\n\t\tBoostedDTables bt = new BoostedDTables();\n\t\tbt.add(dt1);\n\t\tbt.add(dt2);\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\tbt.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\tin.readLine();\n\t\t\t\n\t\t\tBoostedDTables ts = new BoostedDTables();\n\t\t\tts.read(in);\n\t\t\t\n\t\t\tAssert.assertEquals(2, ts.size());\n\t\t\tDecisionTable t1 = ts.get(0);\n\t\t\tAssert.assertArrayEquals(dt1.getAttributeIndices(), t1.getAttributeIndices());\n\t\t\tAssert.assertArrayEquals(dt1.getSplits(), t1.getSplits(), MathUtils.EPSILON);\n\t\t\t\n\t\t\tDecisionTable t2 = ts.get(1);\n\t\t\tAssert.assertArrayEquals(dt2.getAttributeIndices(), t2.getAttributeIndices());\n\t\t\tAssert.assertArrayEquals(dt2.getSplits(), t2.getSplits(), MathUtils.EPSILON);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/BoostedRTreesTest.java",
    "content": "package mltk.predictor.tree.ensemble;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.predictor.tree.RegressionTree;\nimport mltk.predictor.tree.RegressionTreeLeaf;\nimport mltk.predictor.tree.RegressionTreeTestHelper;\nimport mltk.predictor.tree.TreeInteriorNode;\nimport mltk.util.MathUtils;\n\npublic class BoostedRTreesTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tRegressionTree tree1 = RegressionTreeTestHelper.getInstance().getTree1();\n\t\tRegressionTree tree2 = RegressionTreeTestHelper.getInstance().getTree2();\n\t\tBoostedRTrees bt = new BoostedRTrees();\n\t\tbt.add(tree1);\n\t\tbt.add(tree2);\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\tbt.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\tin.readLine();\n\t\t\t\n\t\t\tBoostedRTrees ts = new BoostedRTrees();\n\t\t\tts.read(in);\n\t\t\t\n\t\t\tAssert.assertEquals(2, ts.size());\n\t\t\tAssert.assertTrue(ts.get(0) instanceof RegressionTree);\n\t\t\tAssert.assertTrue(ts.get(1) instanceof RegressionTree);\n\t\t\tRegressionTree t1 = (RegressionTree) ts.get(0);\n\t\t\t\n\t\t\tAssert.assertTrue(t1.getRoot() instanceof TreeInteriorNode);\n\t\t\tTreeInteriorNode root = (TreeInteriorNode) t1.getRoot();\n\t\t\tAssert.assertEquals(1, root.getSplitAttributeIndex());\n\t\t\tAssert.assertEquals(0.5, root.getSplitPoint(), MathUtils.EPSILON);\n\t\t\tAssert.assertTrue(root.getLeftChild() instanceof RegressionTreeLeaf);\n\t\t\tAssert.assertTrue(root.getRightChild() instanceof TreeInteriorNode);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/brt/BDTTest.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Instance;\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\nimport mltk.predictor.io.PredictorReader;\nimport mltk.predictor.tree.DecisionTable;\nimport mltk.predictor.tree.DecisionTableTestHelper;\nimport mltk.predictor.tree.ensemble.BoostedDTables;\nimport mltk.predictor.tree.ensemble.BoostedRTrees;\nimport mltk.util.MathUtils;\n\npublic class BDTTest {\n\t\n\tprivate BDT bdt;\n\t\n\tpublic BDTTest() {\n\t\tDecisionTable dt1 = DecisionTableTestHelper.getInstance().getTable1();\n\t\tDecisionTable dt2 = DecisionTableTestHelper.getInstance().getTable2();\n\t\tBoostedRTrees bt = new BoostedRTrees();\n\t\tbt.add(dt1);\n\t\tbt.add(dt2);\n\t\t\n\t\tbdt = new BDT(1);\n\t\tbdt.tables[0] = new BoostedDTables(bt);\n\t}\n\n\t@Test\n\tpublic void testIO() {\n\t\tDecisionTable dt1 = DecisionTableTestHelper.getInstance().getTable1();\n\t\tDecisionTable dt2 = DecisionTableTestHelper.getInstance().getTable2();\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\tbdt.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\t\n\t\t\tBDT b = PredictorReader.read(in, BDT.class);\n\t\t\tBoostedDTables ts = b.getDecisionTreeList(0);\n\t\t\t\n\t\t\tAssert.assertEquals(2, ts.size());\n\t\t\tDecisionTable t1 = ts.get(0);\n\t\t\tAssert.assertArrayEquals(dt1.getAttributeIndices(), t1.getAttributeIndices());\n\t\t\tAssert.assertArrayEquals(dt1.getSplits(), t1.getSplits(), MathUtils.EPSILON);\n\t\t\t\n\t\t\tDecisionTable t2 = ts.get(1);\n\t\t\tAssert.assertArrayEquals(dt2.getAttributeIndices(), t2.getAttributeIndices());\n\t\t\tAssert.assertArrayEquals(dt2.getSplits(), t2.getSplits(), MathUtils.EPSILON);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\t\n\t@Test\n\tpublic void testRegress() {\n\t\tDecisionTable dt1 = DecisionTableTestHelper.getInstance().getTable1();\n\t\tDecisionTable dt2 = DecisionTableTestHelper.getInstance().getTable2();\n\t\tBoostedRTrees bt = new BoostedRTrees();\n\t\tbt.add(dt1);\n\t\tbt.add(dt2);\n\t\t\n\t\tBRT brt = new BRT(1);\n\t\tbrt.trees[0] = bt;\n\t\t\n\t\tInstances instances = InstancesTestHelper.getInstance().getDenseRegressionDataset();\n\t\tfor (Instance instance : instances) {\n\t\t\tAssert.assertEquals(brt.regress(instance), bdt.regress(instance), MathUtils.EPSILON);\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/brt/BRTTest.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintWriter;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.predictor.io.PredictorReader;\nimport mltk.predictor.tree.RegressionTree;\nimport mltk.predictor.tree.RegressionTreeLeaf;\nimport mltk.predictor.tree.RegressionTreeTestHelper;\nimport mltk.predictor.tree.TreeInteriorNode;\nimport mltk.predictor.tree.ensemble.BoostedRTrees;\nimport mltk.util.MathUtils;\n\npublic class BRTTest {\n\n\t@Test\n\tpublic void testIO() {\n\t\tRegressionTree tree1 = RegressionTreeTestHelper.getInstance().getTree1();\n\t\tRegressionTree tree2 = RegressionTreeTestHelper.getInstance().getTree2();\n\t\tBoostedRTrees bt = new BoostedRTrees();\n\t\tbt.add(tree1);\n\t\tbt.add(tree2);\n\t\t\n\t\tBRT brt = new BRT(1);\n\t\tbrt.trees[0] = bt;\n\t\t\n\t\ttry {\n\t\t\tByteArrayOutputStream boas = new ByteArrayOutputStream();\n\t\t\tPrintWriter out = new PrintWriter(boas);\n\t\t\tbrt.write(out);\n\t\t\tout.close();\n\t\t\t\n\t\t\tByteArrayInputStream bais = new ByteArrayInputStream(boas.toByteArray());\n\t\t\tBufferedReader in = new BufferedReader(new InputStreamReader(bais));\n\t\t\t\n\t\t\tBRT b = PredictorReader.read(in, BRT.class);\n\t\t\tBoostedRTrees ts = b.getRegressionTreeList(0);\n\t\t\t\n\t\t\tAssert.assertEquals(2, ts.size());\n\t\t\tAssert.assertTrue(ts.get(0) instanceof RegressionTree);\n\t\t\tAssert.assertTrue(ts.get(1) instanceof RegressionTree);\n\t\t\tRegressionTree t1 = (RegressionTree) ts.get(0);\n\t\t\t\n\t\t\tAssert.assertTrue(t1.getRoot() instanceof TreeInteriorNode);\n\t\t\tTreeInteriorNode root = (TreeInteriorNode) t1.getRoot();\n\t\t\tAssert.assertEquals(1, root.getSplitAttributeIndex());\n\t\t\tAssert.assertEquals(0.5, root.getSplitPoint(), MathUtils.EPSILON);\n\t\t\tAssert.assertTrue(root.getLeftChild() instanceof RegressionTreeLeaf);\n\t\t\tAssert.assertTrue(root.getRightChild() instanceof TreeInteriorNode);\n\t\t} catch (Exception e) {\n\t\t\tAssert.fail(\"Should not see exception: \" + e.getMessage());\n\t\t}\n\t}\n\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/brt/BRTUtilsTest.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.predictor.tree.RegressionTreeLearner;\nimport mltk.predictor.tree.TreeLearner;\nimport mltk.util.MathUtils;\n\npublic class BRTUtilsTest {\n\n\t@Test\n\tpublic void testParseRegressionTreeLearner1() {\n\t\tString baseLearner = \"rt:l:100\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.NUM_LEAVES_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(100, rtLearner.getMaxNumLeaves());\n\t}\n\t\n\t@Test\n\tpublic void testParseRegressionTreeLearner2() {\n\t\tString baseLearner = \"rt:d:5\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.DEPTH_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(5, rtLearner.getMaxDepth());\n\t}\n\t\n\t@Test\n\tpublic void testParseRegressionTreeLearner3() {\n\t\tString baseLearner = \"rt:a:0.01\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.ALPHA_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(0.01, rtLearner.getAlpha(), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testParseRegressionTreeLearner4() {\n\t\tString baseLearner = \"rt:s:50\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.MIN_LEAF_SIZE_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(50, rtLearner.getMinLeafSize());\n\t}\n\t\n\t@Test\n\tpublic void testParseRobustRegressionTreeLearner1() {\n\t\tString baseLearner = \"rrt:l:100\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.NUM_LEAVES_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(100, rtLearner.getMaxNumLeaves());\n\t}\n\t\n\t@Test\n\tpublic void testParseRobustRegressionTreeLearner2() {\n\t\tString baseLearner = \"rrt:d:5\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.DEPTH_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(5, rtLearner.getMaxDepth());\n\t}\n\t\n\t@Test\n\tpublic void testParseRobustRegressionTreeLearner3() {\n\t\tString baseLearner = \"rrt:a:0.01\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.ALPHA_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(0.01, rtLearner.getAlpha(), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testParseRobustRegressionTreeLearner4() {\n\t\tString baseLearner = \"rrt:s:50\";\n\t\tTreeLearner treeLearner = null;\n\t\ttreeLearner = BRTUtils.parseTreeLearner(baseLearner);\n\t\tAssert.assertTrue(treeLearner instanceof RegressionTreeLearner);\n\t\tRegressionTreeLearner rtLearner = (RegressionTreeLearner) treeLearner;\n\t\tAssert.assertEquals(RegressionTreeLearner.Mode.MIN_LEAF_SIZE_LIMITED, rtLearner.getConstructionMode());\n\t\tAssert.assertEquals(50, rtLearner.getMinLeafSize());\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/predictor/tree/ensemble/brt/LogitBoostLearnerTest.java",
    "content": "package mltk.predictor.tree.ensemble.brt;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\nimport mltk.core.Instances;\nimport mltk.core.InstancesTestHelper;\nimport mltk.predictor.evaluation.Evaluator;\nimport mltk.predictor.evaluation.MetricFactory;\nimport mltk.predictor.tree.TreeLearner;\n\npublic class LogitBoostLearnerTest {\n\n\t@Test\n\tpublic void testLogitBoostLearner() {\n\t\tTreeLearner treeLearner = BRTUtils.parseTreeLearner(\"rrt:d:3\");\n\t\tInstances instances = InstancesTestHelper.getInstance().getDenseClassificationDataset();\n\t\t\n\t\tLogitBoostLearner learner = new LogitBoostLearner();\n\t\tlearner.setLearningRate(0.1);\n\t\tlearner.setMetric(MetricFactory.getMetric(\"auc\"));\n\t\tlearner.setTreeLearner(treeLearner);\n\t\t\n\t\tBRT brt = learner.buildBinaryClassifier(instances, 10);\n\t\tdouble auc = Evaluator.evalAreaUnderROC(brt, instances);\n\t\tAssert.assertTrue(auc > 0.5);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/util/ArrayUtilsTest.java",
    "content": "package mltk.util;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class ArrayUtilsTest {\n\n\t@Test\n\tpublic void testParseDoubleArray() {\n\t\tString str = \"[1.1, 2.2, 3.3, 4.4]\";\n\t\tdouble[] a = {1.1, 2.2, 3.3, 4.4};\n\t\tAssert.assertArrayEquals(a, ArrayUtils.parseDoubleArray(str), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testParseIntArray() {\n\t\tString str = \"[1, 2, 3, 4]\";\n\t\tint[] a = {1, 2, 3, 4};\n\t\tAssert.assertArrayEquals(a, ArrayUtils.parseIntArray(str));\n\t}\n\t\n\t@Test\n\tpublic void testIsConstant() {\n\t\tint[] a = {1, 1, 1};\n\t\tint[] b = {2, 1, 1};\n\t\tAssert.assertTrue(ArrayUtils.isConstant(a, 0, a.length, 1));\n\t\tAssert.assertFalse(ArrayUtils.isConstant(b, 0, b.length, 1));\n\t\tAssert.assertTrue(ArrayUtils.isConstant(b, 1, b.length, 1));\n\t\t\n\t\tdouble[] c = {0.1, 0.1, 0.1, 0.1};\n\t\tdouble[] d = {0.2, 0.1, 0.1, 0.1};\n\t\tAssert.assertTrue(ArrayUtils.isConstant(c, 0, c.length, 0.1));\n\t\tAssert.assertFalse(ArrayUtils.isConstant(d, 0, d.length, 0.1));\n\t\tAssert.assertTrue(ArrayUtils.isConstant(d, 1, d.length, 0.1));\n\t}\n\t\n\t@Test\n\tpublic void testGetMedian() {\n\t\tdouble[] a = {0.7, 0.4, 0.3, 0.2, 0.5, 0.6, 0.1};\n\t\tAssert.assertEquals(0.4, ArrayUtils.getMedian(a), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/util/MathUtilsTest.java",
    "content": "package mltk.util;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class MathUtilsTest {\n\t\n\t@Test\n\tpublic void testEquals() {\n\t\tAssert.assertTrue(MathUtils.equals(0.1, 0.10000001));\n\t\tAssert.assertFalse(MathUtils.equals(0.0, 1.0));\n\t}\n\n\t@Test\n\tpublic void testIndicator() {\n\t\tAssert.assertEquals(1, MathUtils.indicator(true));\n\t\tAssert.assertEquals(0, MathUtils.indicator(false));\n\t}\n\t\n\t@Test\n\tpublic void testIsFirstBetter() {\n\t\tAssert.assertTrue(MathUtils.isFirstBetter(0.5, 0, true));\n\t\tAssert.assertFalse(MathUtils.isFirstBetter(0.5, 0, false));\n\t}\n\t\n\t@Test\n\tpublic void testIsInteger() {\n\t\tAssert.assertTrue(MathUtils.isInteger(1.0));\n\t\tAssert.assertFalse(MathUtils.isInteger(1.1));\n\t}\n\t\n\t@Test\n\tpublic void testIsZero() {\n\t\tAssert.assertTrue(MathUtils.isZero(MathUtils.EPSILON / 2));\n\t\tAssert.assertFalse(MathUtils.isZero(MathUtils.EPSILON * 2));\n\t}\n\t\n\t@Test\n\tpublic void testSigmoid() {\n\t\tAssert.assertEquals(0.5, MathUtils.sigmoid(0), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testSign() {\n\t\tAssert.assertEquals(1, MathUtils.sign(0.5));\n\t\tAssert.assertEquals(0, MathUtils.sign(0.0));\n\t\tAssert.assertEquals(-1, MathUtils.sign(-0.5));\n\t\t\n\t\tAssert.assertEquals(1, MathUtils.sign(2));\n\t\tAssert.assertEquals(0, MathUtils.sign(0));\n\t\tAssert.assertEquals(-1, MathUtils.sign(-2));\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/util/OptimUtilsTest.java",
    "content": "package mltk.util;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class OptimUtilsTest {\n\n\t@Test\n\tpublic void testGetProbability() {\n\t\tAssert.assertEquals(0.5, OptimUtils.getProbability(0), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testGetResidual() {\n\t\tAssert.assertEquals(-1.0, OptimUtils.getResidual(1.0, 0), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testGetPseudoResidual() {\n\t\tAssert.assertEquals(0.5, OptimUtils.getPseudoResidual(0, 1), MathUtils.EPSILON);\n\t\tAssert.assertEquals(-0.5, OptimUtils.getPseudoResidual(0, 0), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testComputeLogisticLoss() {\n\t\tAssert.assertEquals(0.693147181, OptimUtils.computeLogisticLoss(0, 1), MathUtils.EPSILON);\n\t\tAssert.assertEquals(0.693147181, OptimUtils.computeLogisticLoss(0, -1), MathUtils.EPSILON);\n\t\tAssert.assertEquals(0.006715348, OptimUtils.computeLogisticLoss(5, 1), MathUtils.EPSILON);\n\t\tAssert.assertEquals(5.006715348, OptimUtils.computeLogisticLoss(5, -1), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testIsConverged() {\n\t\tAssert.assertTrue(OptimUtils.isConverged(0.100000001, 0.1, 1e-6));\n\t\tAssert.assertFalse(OptimUtils.isConverged(0.15, 0.1, 1e-6));\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/util/StatUtilsTest.java",
    "content": "package mltk.util;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class StatUtilsTest {\n\t\n\tprivate int[] a = {1, 4, 3, 2};\n\tprivate double[] b = {-1.2, 1.2, -5.3, 5.3};\n\n\t@Test\n\tpublic void testMax() {\n\t\tAssert.assertEquals(4, StatUtils.max(a));\n\t\tAssert.assertEquals(5.3, StatUtils.max(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testIndexOfMax() {\n\t\tAssert.assertEquals(1, StatUtils.indexOfMax(a));\n\t\tAssert.assertEquals(3, StatUtils.indexOfMax(b));\n\t}\n\t\n\t@Test\n\tpublic void testMin() {\n\t\tAssert.assertEquals(1, StatUtils.min(a));\n\t\tAssert.assertEquals(-5.3, StatUtils.min(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testIndexOfMin() {\n\t\tAssert.assertEquals(0, StatUtils.indexOfMin(a));\n\t\tAssert.assertEquals(2, StatUtils.indexOfMin(b));\n\t}\n\t\n\t@Test\n\tpublic void testSum() {\n\t\tAssert.assertEquals(0, StatUtils.sum(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testSumSq() {\n\t\tAssert.assertEquals(59.06, StatUtils.sumSq(b), MathUtils.EPSILON);\n\t\tAssert.assertEquals(b[0] * b[0], StatUtils.sumSq(b, 0, 1), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testMean() {\n\t\tAssert.assertEquals(0, StatUtils.mean(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testVariance() {\n\t\tAssert.assertEquals(19.686666667, StatUtils.variance(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testStd() {\n\t\tAssert.assertEquals(Math.sqrt(19.686666667), StatUtils.sd(b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testRms() {\n\t\tAssert.assertEquals(Math.sqrt(59.06 / b.length), StatUtils.rms(b), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  },
  {
    "path": "src/test/java/mltk/util/VectorUtilsTest.java",
    "content": "package mltk.util;\n\nimport org.junit.Assert;\nimport org.junit.Test;\n\npublic class VectorUtilsTest {\n\n\t@Test\n\tpublic void testAdd() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {2, 3, 4, 5};\n\t\tVectorUtils.add(a, 1);\n\t\tAssert.assertArrayEquals(b, a, MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testSubtract() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {2, 3, 4, 5};\n\t\tVectorUtils.subtract(b, 1);\n\t\tAssert.assertArrayEquals(a, b, MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testMultiply() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {2, 4, 6, 8};\n\t\tVectorUtils.multiply(a, 2);\n\t\tAssert.assertArrayEquals(b, a, MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testDivide() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {2, 4, 6, 8};\n\t\tVectorUtils.divide(b, 2);\n\t\tAssert.assertArrayEquals(a, b, MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testL2norm() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tAssert.assertEquals(5.477225575, VectorUtils.l2norm(a), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testL1norm() {\n\t\tdouble[] a = {1, -2, 3, -4};\n\t\tAssert.assertEquals(10, VectorUtils.l1norm(a), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testDotProduct() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {0, -1, 0, 1};\n\t\tAssert.assertEquals(2, VectorUtils.dotProduct(a, b), MathUtils.EPSILON);\n\t}\n\t\n\t@Test\n\tpublic void testCorrelation() {\n\t\tdouble[] a = {1, 2, 3, 4};\n\t\tdouble[] b = {2, 4, 6, 8};\n\t\tdouble[] c = {-2, -4, -6, -8};\n\t\tAssert.assertEquals(1, VectorUtils.correlation(a, b), MathUtils.EPSILON);\n\t\tAssert.assertEquals(-1, VectorUtils.correlation(a, c), MathUtils.EPSILON);\n\t}\n\t\n}\n"
  }
]