entry : list) {\n result.put(entry.getKey(), entry.getValue());\n }\n return result;\n }\n\n /**\n * Sample a value from a double array\n * \n * @param probs\n * @return\n */\n public static int nextDiscrete(double[] probs)\n {\n double sum = 0.0;\n for (int i = 0; i < probs.length; i++)\n sum += probs[i];\n\n double r = MTRandom.nextDouble() * sum;\n\n sum = 0.0;\n for (int i = 0; i < probs.length; i++) {\n sum += probs[i];\n if (sum > r)\n return i;\n }\n return probs.length - 1;\n }\n\n public static double mean(double[] m)\n {\n double sum = 0;\n for (int i = 0; i < m.length; i++)\n sum += m[i];\n return sum / m.length;\n }\n\n public static double stddev(double[] m)\n {\n double mean = mean(m);\n double s = 0;\n for (int i = 0; i < m.length; i++)\n s += (m[i] - mean) * (m[i] - mean);\n return Math.sqrt(s / m.length);\n }\n}\n"
},
{
"path": "src/utility/LBFGS.java",
"content": "/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.\n This file is part of \"MALLET\" (MAchine Learning for LanguagE Toolkit).\n http://www.cs.umass.edu/~mccallum/mallet\n This software is provided under the terms of the Common Public License,\n version 1.0, as published by http://www.opensource.org. For further\n information, see the file `LICENSE' included with this distribution. */\n\n/** \n @author Aron Culotta culotta@cs.umass.edu\n */\n\n/**\n Limited Memory BFGS, as described in Byrd, Nocedal, and Schnabel,\n \"Representations of Quasi-Newton Matrices and Their Use in Limited\n Memory Methods\"\n */\npackage utility;\n\nimport java.util.LinkedList;\nimport java.util.logging.Logger;\n\nimport cc.mallet.optimize.BackTrackLineSearch;\nimport cc.mallet.optimize.InvalidOptimizableException;\nimport cc.mallet.optimize.LineOptimizer;\nimport cc.mallet.optimize.Optimizable;\nimport cc.mallet.optimize.Optimizer;\nimport cc.mallet.optimize.OptimizerEvaluator;\nimport cc.mallet.types.MatrixOps;\nimport cc.mallet.util.MalletLogger;\n\npublic class LBFGS\n implements Optimizer\n{\n private static Logger logger = MalletLogger\n .getLogger(\"edu.umass.cs.mallet.base.ml.maximize.LimitedMemoryBFGS\");\n\n boolean converged = false;\n Optimizable.ByGradientValue optimizable;\n final int maxIterations = 5000;\n // xxx need a more principled stopping point\n // final double tolerance = .0001;\n // private double tolerance = 1.0e-10;\n // final double gradientTolerance = 1.0e-10;\n\n double tolerance;// = 1.0e-3;\n final double gradientTolerance = 1.0e-5;\n final double eps = 1.0e-10;\n\n // The number of corrections used in BFGS update\n // ideally 3 <= m <= 7. Larger m means more cpu time, memory.\n final int m = 10;\n\n // Line search function\n private LineOptimizer.ByGradient lineMaximizer;\n\n public LBFGS(Optimizable.ByGradientValue function, double inTolerance)\n {\n tolerance = inTolerance;\n this.optimizable = function;\n lineMaximizer = new BackTrackLineSearch(function);\n }\n\n @Override\n public Optimizable getOptimizable()\n {\n return this.optimizable;\n }\n\n @Override\n public boolean isConverged()\n {\n return converged;\n }\n\n /**\n * Sets the LineOptimizer.ByGradient to use in L-BFGS optimization.\n * \n * @param lineOpt\n * line optimizer for L-BFGS\n */\n public void setLineOptimizer(LineOptimizer.ByGradient lineOpt)\n {\n lineMaximizer = lineOpt;\n }\n\n // State of search\n // g = gradient\n // s = list of m previous \"parameters\" values\n // y = list of m previous \"g\" values\n // rho = intermediate calculation\n double[] g, oldg, direction, parameters, oldParameters;\n LinkedList s = new LinkedList();\n LinkedList y = new LinkedList();\n LinkedList rho = new LinkedList();\n double[] alpha;\n static double step = 1.0;\n int iterations;\n\n private OptimizerEvaluator.ByGradient eval = null;\n\n // CPAL - added this\n public void setTolerance(double newtol)\n {\n this.tolerance = newtol;\n }\n\n public void setEvaluator(OptimizerEvaluator.ByGradient eval)\n {\n this.eval = eval;\n }\n\n public int getIteration()\n {\n return iterations;\n }\n\n @Override\n public boolean optimize()\n {\n return optimize(Integer.MAX_VALUE);\n }\n\n @Override\n public boolean optimize(int numIterations)\n {\n\n double initialValue = optimizable.getValue();\n logger.fine(\"Entering L-BFGS.optimize(). Initial Value=\" + initialValue);\n\n if (g == null) { // first time through\n logger.fine(\"First time through L-BFGS\");\n iterations = 0;\n s = new LinkedList();\n y = new LinkedList();\n rho = new LinkedList();\n alpha = new double[m];\n for (int i = 0; i < m; i++)\n alpha[i] = 0.0;\n\n parameters = new double[optimizable.getNumParameters()];\n oldParameters = new double[optimizable.getNumParameters()];\n g = new double[optimizable.getNumParameters()];\n oldg = new double[optimizable.getNumParameters()];\n direction = new double[optimizable.getNumParameters()];\n\n optimizable.getParameters(parameters);\n System.arraycopy(parameters, 0, oldParameters, 0, parameters.length);\n\n optimizable.getValueGradient(g);\n System.arraycopy(g, 0, oldg, 0, g.length);\n System.arraycopy(g, 0, direction, 0, g.length);\n\n if (MatrixOps.absNormalize(direction) == 0) {\n logger.info(\"L-BFGS initial gradient is zero; saying converged\");\n g = null;\n converged = true;\n return true;\n }\n logger.fine(\"direction.2norm: \" + MatrixOps.twoNorm(direction));\n MatrixOps.timesEquals(direction, 1.0 / MatrixOps.twoNorm(direction));\n // make initial jump\n logger.fine(\"before initial jump: \\ndirection.2norm: \" + MatrixOps.twoNorm(direction)\n + \" \\ngradient.2norm: \" + MatrixOps.twoNorm(g) + \"\\nparameters.2norm: \"\n + MatrixOps.twoNorm(parameters));\n\n // TestMaximizable.testValueAndGradientInDirection (maxable,\n // direction);\n step = lineMaximizer.optimize(direction, step);\n if (step == 0.0) {// could not step in this direction.\n // // give up and say converged.\n // g = null; // reset search\n // step = 1.0;\n // throw new OptimizationException(\n // \"Line search could not step in the current direction. \"\n // +\n // \"(This is not necessarily cause for alarm. Sometimes this happens close to the maximum,\"\n // + \" where the function may be very flat.)\");\n\n return false;\n }\n optimizable.getParameters(parameters);\n optimizable.getValueGradient(g);\n logger.fine(\"after initial jump: \\ndirection.2norm: \" + MatrixOps.twoNorm(direction)\n + \" \\ngradient.2norm: \" + MatrixOps.twoNorm(g));\n }\n\n double value = optimizable.getValue();\n\n for (int iterationCount = 0; iterationCount < numIterations; iterationCount++) {\n\n logger.fine(\"L-BFGS iteration=\" + iterationCount + \", value=\" + value + \" g.twoNorm: \"\n + MatrixOps.twoNorm(g) + \" oldg.twoNorm: \" + MatrixOps.twoNorm(oldg));\n\n // if (iterationCount % 10 == 0)\n // System.out.println(\"\\t\\tL-BFGS iteration=\" + iterationCount\n // + \", value=\" + value + \" g.twoNorm: \"\n // + MatrixOps.twoNorm(g) + \" oldg.twoNorm: \"\n // + MatrixOps.twoNorm(oldg));\n\n // get difference between previous 2 gradients and parameters\n double sy = 0.0;\n double yy = 0.0;\n for (int i = 0; i < oldParameters.length; i++) {\n // -inf - (-inf) = 0; inf - inf = 0\n if (Double.isInfinite(parameters[i]) && Double.isInfinite(oldParameters[i])\n && (parameters[i] * oldParameters[i] > 0))\n oldParameters[i] = 0.0;\n else\n oldParameters[i] = parameters[i] - oldParameters[i];\n if (Double.isInfinite(g[i]) && Double.isInfinite(oldg[i]) && (g[i] * oldg[i] > 0))\n oldg[i] = 0.0;\n else\n oldg[i] = g[i] - oldg[i];\n sy += oldParameters[i] * oldg[i]; // si * yi\n yy += oldg[i] * oldg[i];\n direction[i] = g[i];\n }\n\n if (sy > 0) {\n throw new InvalidOptimizableException(\"sy = \" + sy + \" > 0\");\n }\n\n double gamma = sy / yy; // scaling factor\n if (gamma > 0)\n throw new InvalidOptimizableException(\"gamma = \" + gamma + \" > 0\");\n\n push(rho, 1.0 / sy);\n push(s, oldParameters);\n push(y, oldg);\n // calculate new direction\n assert (s.size() == y.size()) : \"s.size: \" + s.size() + \" y.size: \" + y.size();\n for (int i = s.size() - 1; i >= 0; i--) {\n alpha[i] = ((Double) rho.get(i)).doubleValue()\n * MatrixOps.dotProduct((double[]) s.get(i), direction);\n MatrixOps.plusEquals(direction, (double[]) y.get(i), -1.0 * alpha[i]);\n }\n MatrixOps.timesEquals(direction, gamma);\n for (int i = 0; i < y.size(); i++) {\n double beta = (((Double) rho.get(i)).doubleValue())\n * MatrixOps.dotProduct((double[]) y.get(i), direction);\n MatrixOps.plusEquals(direction, (double[]) s.get(i), alpha[i] - beta);\n }\n\n for (int i = 0; i < oldg.length; i++) {\n oldParameters[i] = parameters[i];\n oldg[i] = g[i];\n direction[i] *= -1.0;\n }\n logger.fine(\"before linesearch: direction.gradient.dotprod: \"\n + MatrixOps.dotProduct(direction, g) + \"\\ndirection.2norm: \"\n + MatrixOps.twoNorm(direction) + \"\\nparameters.2norm: \"\n + MatrixOps.twoNorm(parameters));\n // TestMaximizable.testValueAndGradientInDirection (maxable,\n // direction);\n step = lineMaximizer.optimize(direction, step);\n if (step == 0.0) { // could not step in this direction.\n g = null; // reset search\n step = 1.0;\n // xxx Temporary test; passed OK\n // TestMaximizable.testValueAndGradientInDirection (maxable,\n // direction);\n // System.out\n // .println(\"\\t\\tLine search could not step in the current direction.\");\n // throw new OptimizationException(\n // \"Line search could not step in the current direction. \"\n // +\n // \"(This is not necessarily cause for alarm. Sometimes this happens close to the maximum,\"\n // + \" where the function may be very flat.)\");\n return false;\n }\n optimizable.getParameters(parameters);\n optimizable.getValueGradient(g);\n logger.fine(\"after linesearch: direction.2norm: \" + MatrixOps.twoNorm(direction));\n double newValue = optimizable.getValue();\n\n // Test for terminations\n // if(2.0*Math.abs(newValue-value) <= tolerance*\n // (Math.abs(newValue)+Math.abs(value) + eps)){\n if (Math.abs(newValue - value) <= tolerance) {\n // System.out.println(\"\\t\\tNumber of iterations: \"\n // + iterationCount);\n // System.out\n // .println(\"\\t\\tExiting L-BFGS on termination #1:\\n\\t\\tvalue difference below \"\n // + tolerance\n // + \" (oldValue: \"\n // + value\n // + \" newValue: \"\n // + newValue\n // + \" gradient.twoNorm: \"\n // + MatrixOps.twoNorm(g) + \")\");\n converged = true;\n return true;\n }\n\n value = newValue;\n\n double gg = MatrixOps.twoNorm(g);\n if (gg < gradientTolerance) {\n logger.fine(\"Exiting L-BFGS on termination #2: \\ngradient=\" + gg + \" < \"\n + gradientTolerance);\n converged = true;\n return true;\n }\n if (gg == 0.0) {\n logger.fine(\"Exiting L-BFGS on termination #3: \\ngradient==0.0\");\n converged = true;\n return true;\n }\n logger.fine(\"Gradient = \" + gg);\n iterations++;\n if (iterations > maxIterations) {\n System.err\n .println(\"Too many iterations in L-BFGS.java. Continuing with current parameters.\");\n converged = true;\n return true;\n // throw new IllegalStateException (\"Too many iterations.\");\n }\n\n // end of iteration. call evaluator\n if (eval != null && !eval.evaluate(optimizable, iterationCount)) {\n logger.fine(\"Exiting L-BFGS on termination #4: evaluator returned false.\");\n converged = true;\n return false;\n }\n }\n return false;\n }\n\n /**\n * Resets the previous gradients and values that are used to approximate the Hessian. NOTE - If\n * the {@link Optimizable} object is modified externally, this method should be called to avoid\n * IllegalStateExceptions.\n */\n public void reset()\n {\n g = null;\n }\n\n /**\n * Pushes a new object onto the queue l\n * \n * @param l\n * linked list queue of Matrix obj's\n * @param toadd\n * matrix to push onto queue\n */\n private void push(LinkedList l, double[] toadd)\n {\n assert (l.size() <= m);\n if (l.size() == m) {\n // remove oldest matrix and add newset to end of list.\n // to make this more efficient, actually overwrite\n // memory of oldest matrix\n\n // this overwrites the oldest matrix\n double[] last = (double[]) l.get(0);\n System.arraycopy(toadd, 0, last, 0, toadd.length);\n Object ptr = last;\n // this readjusts the pointers in the list\n for (int i = 0; i < l.size() - 1; i++)\n l.set(i, l.get(i + 1));\n l.set(m - 1, ptr);\n }\n else {\n double[] newArray = new double[toadd.length];\n System.arraycopy(toadd, 0, newArray, 0, toadd.length);\n l.addLast(newArray);\n }\n }\n\n /**\n * Pushes a new object onto the queue l\n * \n * @param l\n * linked list queue of Double obj's\n * @param toadd\n * double value to push onto queue\n */\n private void push(LinkedList l, double toadd)\n {\n assert (l.size() <= m);\n if (l.size() == m) { // pop old double and add new\n l.removeFirst();\n l.addLast(new Double(toadd));\n }\n else\n l.addLast(new Double(toadd));\n }\n\n}\n"
},
{
"path": "src/utility/MTRandom.java",
"content": "package utility;\n\npublic class MTRandom\n{\n\n private static MersenneTwister rand = new MersenneTwister();\n\n public static void setSeed(long seed)\n {\n rand.setSeed(seed);\n }\n\n public static double nextDouble()\n {\n return rand.nextDouble();\n }\n\n public static int nextInt(int n)\n {\n return rand.nextInt(n);\n }\n\n public static boolean nextBoolean()\n {\n return rand.nextBoolean();\n }\n}\n"
},
{
"path": "src/utility/MersenneTwister.java",
"content": "package utility;\n\nimport java.io.DataInputStream;\nimport java.io.DataOutputStream;\nimport java.io.IOException;\nimport java.io.ObjectInputStream;\nimport java.io.ObjectOutputStream;\nimport java.io.Serializable;\n\n/**\n * MersenneTwister and MersenneTwisterFast
\n * \n * Version 20, based on version MT199937(99/10/29) of the Mersenne Twister algorithm found at\n * The Mersenne Twister Home Page, with\n * the initialization improved using the new 2002/1/26 initialization algorithm By Sean Luke,\n * October 2004.\n * \n *
\n * MersenneTwister is a drop-in subclass replacement for java.util.Random. It is properly\n * synchronized and can be used in a multithreaded environment. On modern VMs such as HotSpot, it is\n * approximately 1/3 slower than java.util.Random.\n *\n *
\n * MersenneTwisterFast is not a subclass of java.util.Random. It has the same public methods\n * as Random does, however, and it is algorithmically identical to MersenneTwister.\n * MersenneTwisterFast has hard-code inlined all of its methods directly, and made all of them final\n * (well, the ones of consequence anyway). Further, these methods are not synchronized, so\n * the same MersenneTwisterFast instance cannot be shared by multiple threads. But all this helps\n * MersenneTwisterFast achieve well over twice the speed of MersenneTwister. java.util.Random is\n * about 1/3 slower than MersenneTwisterFast.\n *\n *
About the Mersenne Twister
\n * \n * This is a Java version of the C-program for MT19937: Integer version. The MT19937 algorithm was\n * created by Makoto Matsumoto and Takuji Nishimura, who ask: \"When you use this, send an email to:\n * matumoto@math.keio.ac.jp with an appropriate reference to your work\". Indicate that this is a\n * translation of their algorithm into Java.\n *\n *
\n * Reference. Makato Matsumoto and Takuji Nishimura, \"Mersenne Twister: A 623-Dimensionally\n * Equidistributed Uniform Pseudo-Random Number Generator\", ACM Transactions on Modeling and.\n * Computer Simulation, Vol. 8, No. 1, January 1998, pp 3--30.\n *\n *
About this Version
\n *\n * \n * Changes since V19: nextFloat(boolean, boolean) now returns float, not double.\n *\n *
\n * Changes since V18: Removed old final declarations, which used to potentially speed up the\n * code, but no longer.\n *\n *
\n * Changes since V17: Removed vestigial references to &= 0xffffffff which stemmed from the\n * original C code. The C code could not guarantee that ints were 32 bit, hence the masks. The\n * vestigial references in the Java code were likely optimized out anyway.\n *\n *
\n * Changes since V16: Added nextDouble(includeZero, includeOne) and nextFloat(includeZero,\n * includeOne) to allow for half-open, fully-closed, and fully-open intervals.\n *\n *
\n * Changes Since V15: Added serialVersionUID to quiet compiler warnings from Sun's overly\n * verbose compilers as of JDK 1.5.\n *\n *
\n * Changes Since V14: made strictfp, with StrictMath.log and StrictMath.sqrt in nextGaussian\n * instead of Math.log and Math.sqrt. This is largely just to be safe, as it presently makes no\n * difference in the speed, correctness, or results of the algorithm.\n *\n *
\n * Changes Since V13: clone() method CloneNotSupportedException removed.\n *\n *
\n * Changes Since V12: clone() method added.\n *\n *
\n * Changes Since V11: stateEquals(...) method added. MersenneTwisterFast is equal to other\n * MersenneTwisterFasts with identical state; likewise MersenneTwister is equal to other\n * MersenneTwister with identical state. This isn't equals(...) because that requires a contract of\n * immutability to compare by value.\n *\n *
\n * Changes Since V10: A documentation error suggested that setSeed(int[]) required an int[]\n * array 624 long. In fact, the array can be any non-zero length. The new version also checks for\n * this fact.\n *\n *
\n * Changes Since V9: readState(stream) and writeState(stream) provided.\n *\n *
\n * Changes Since V8: setSeed(int) was only using the first 28 bits of the seed; it should\n * have been 32 bits. For small-number seeds the behavior is identical.\n *\n *
\n * Changes Since V7: A documentation error in MersenneTwisterFast (but not MersenneTwister)\n * stated that nextDouble selects uniformly from the full-open interval [0,1]. It does not.\n * nextDouble's contract is identical across MersenneTwisterFast, MersenneTwister, and\n * java.util.Random, namely, selection in the half-open interval [0,1). That is, 1.0 should not be\n * returned. A similar contract exists in nextFloat.\n *\n *
\n * Changes Since V6: License has changed from LGPL to BSD. New timing information to compare\n * against java.util.Random. Recent versions of HotSpot have helped Random increase in speed to the\n * point where it is faster than MersenneTwister but slower than MersenneTwisterFast (which should\n * be the case, as it's a less complex algorithm but is synchronized).\n * \n *
\n * Changes Since V5: New empty constructor made to work the same as java.util.Random --\n * namely, it seeds based on the current time in milliseconds.\n *\n *
\n * Changes Since V4: New initialization algorithms. See (see \n * http://www.math.keio.ac.jp/matumoto/MT2002/emt19937ar.html)\n *\n * \n * The MersenneTwister code is based on standard MT19937 C/C++ code by Takuji Nishimura, with\n * suggestions from Topher Cooper and Marc Rieffel, July 1997. The code was originally translated\n * into Java by Michael Lecuyer, January 1999, and the original code is Copyright (c) 1999 by\n * Michael Lecuyer.\n *\n *
Java notes
\n * \n * \n * This implementation implements the bug fixes made in Java 1.2's version of Random, which means it\n * can be used with earlier versions of Java. See the JDK 1.2\n * java.util.Random documentation for further documentation on the random-number generation\n * contracts made. Additionally, there's an undocumented bug in the JDK java.util.Random.nextBytes()\n * method, which this code fixes.\n *\n * \n * Just like java.util.Random, this generator accepts a long seed but doesn't use all of it.\n * java.util.Random uses 48 bits. The Mersenne Twister instead uses 32 bits (int size). So it's best\n * if your seed does not exceed the int range.\n *\n *
\n * MersenneTwister can be used reliably on JDK version 1.1.5 or above. Earlier Java versions have\n * serious bugs in java.util.Random; only MersenneTwisterFast (and not MersenneTwister nor\n * java.util.Random) should be used with them.\n *\n *
License
\n *\n * Copyright (c) 2003 by Sean Luke.
\n * Portions copyright (c) 1993 by Michael Lecuyer.
\n * All rights reserved.
\n *\n * \n * Redistribution and use in source and binary forms, with or without modification, are permitted\n * provided that the following conditions are met:\n *
\n * - Redistributions of source code must retain the above copyright notice, this list of\n * conditions and the following disclaimer.\n *
- Redistributions in binary form must reproduce the above copyright notice, this list of\n * conditions and the following disclaimer in the documentation and/or other materials provided with\n * the distribution.\n *
- Neither the name of the copyright owners, their employers, nor the names of its contributors\n * may be used to endorse or promote products derived from this software without specific prior\n * written permission.\n *
\n * \n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\n * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR\n * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,\n * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY\n * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n * @version 20\n */\n\npublic strictfp class MersenneTwister\n extends java.util.Random\n implements Serializable, Cloneable\n{\n // Serialization\n private static final long serialVersionUID = -4035832775130174188L; // locked as of Version 15\n\n // Period parameters\n private static final int N = 624;\n private static final int M = 397;\n private static final int MATRIX_A = 0x9908b0df; // private static final * constant vector a\n private static final int UPPER_MASK = 0x80000000; // most significant w-r bits\n private static final int LOWER_MASK = 0x7fffffff; // least significant r bits\n\n // Tempering parameters\n private static final int TEMPERING_MASK_B = 0x9d2c5680;\n private static final int TEMPERING_MASK_C = 0xefc60000;\n\n private int mt[]; // the array for the state vector\n private int mti; // mti==N+1 means mt[N] is not initialized\n private int mag01[];\n\n // a good initial seed (of int size, though stored in a long)\n // private static final long GOOD_SEED = 4357;\n\n /*\n * implemented here because there's a bug in Random's implementation of the Gaussian code\n * (divide by zero, and log(0), ugh!), yet its gaussian variables are private so we can't access\n * them here. :-(\n */\n\n private double __nextNextGaussian;\n private boolean __haveNextNextGaussian;\n\n /* We're overriding all internal data, to my knowledge, so this should be okay */\n public Object clone()\n {\n try {\n MersenneTwister f = (MersenneTwister) (super.clone());\n f.mt = (int[]) (mt.clone());\n f.mag01 = (int[]) (mag01.clone());\n return f;\n }\n catch (CloneNotSupportedException e) {\n throw new InternalError();\n } // should never happen\n }\n\n public boolean stateEquals(Object o)\n {\n if (o == this)\n return true;\n if (o == null || !(o instanceof MersenneTwister))\n return false;\n MersenneTwister other = (MersenneTwister) o;\n if (mti != other.mti)\n return false;\n for (int x = 0; x < mag01.length; x++)\n if (mag01[x] != other.mag01[x])\n return false;\n for (int x = 0; x < mt.length; x++)\n if (mt[x] != other.mt[x])\n return false;\n return true;\n }\n\n /** Reads the entire state of the MersenneTwister RNG from the stream */\n public void readState(DataInputStream stream)\n throws IOException\n {\n int len = mt.length;\n for (int x = 0; x < len; x++)\n mt[x] = stream.readInt();\n\n len = mag01.length;\n for (int x = 0; x < len; x++)\n mag01[x] = stream.readInt();\n\n mti = stream.readInt();\n __nextNextGaussian = stream.readDouble();\n __haveNextNextGaussian = stream.readBoolean();\n }\n\n /** Writes the entire state of the MersenneTwister RNG to the stream */\n public void writeState(DataOutputStream stream)\n throws IOException\n {\n int len = mt.length;\n for (int x = 0; x < len; x++)\n stream.writeInt(mt[x]);\n\n len = mag01.length;\n for (int x = 0; x < len; x++)\n stream.writeInt(mag01[x]);\n\n stream.writeInt(mti);\n stream.writeDouble(__nextNextGaussian);\n stream.writeBoolean(__haveNextNextGaussian);\n }\n\n /**\n * Constructor using the default seed.\n */\n public MersenneTwister()\n {\n this(System.currentTimeMillis());\n }\n\n /**\n * Constructor using a given seed. Though you pass this seed in as a long, it's best to make\n * sure it's actually an integer.\n */\n public MersenneTwister(long seed)\n {\n super(seed); /* just in case */\n setSeed(seed);\n }\n\n /**\n * Constructor using an array of integers as seed. Your array must have a non-zero length. Only\n * the first 624 integers in the array are used; if the array is shorter than this then integers\n * are repeatedly used in a wrap-around fashion.\n */\n public MersenneTwister(int[] array)\n {\n super(System.currentTimeMillis()); /* pick something at random just in case */\n setSeed(array);\n }\n\n /**\n * Initalize the pseudo random number generator. Don't pass in a long that's bigger than an int\n * (Mersenne Twister only uses the first 32 bits for its seed).\n */\n\n synchronized public void setSeed(long seed)\n {\n // it's always good style to call super\n super.setSeed(seed);\n\n // Due to a bug in java.util.Random clear up to 1.2, we're\n // doing our own Gaussian variable.\n __haveNextNextGaussian = false;\n\n mt = new int[N];\n\n mag01 = new int[2];\n mag01[0] = 0x0;\n mag01[1] = MATRIX_A;\n\n mt[0] = (int) (seed & 0xffffffff);\n mt[0] = (int) seed;\n for (mti = 1; mti < N; mti++) {\n mt[mti] = (1812433253 * (mt[mti - 1] ^ (mt[mti - 1] >>> 30)) + mti);\n /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */\n /* In the previous versions, MSBs of the seed affect */\n /* only MSBs of the array mt[]. */\n /* 2002/01/09 modified by Makoto Matsumoto */\n // mt[mti] &= 0xffffffff;\n /* for >32 bit machines */\n }\n }\n\n /**\n * Sets the seed of the MersenneTwister using an array of integers. Your array must have a\n * non-zero length. Only the first 624 integers in the array are used; if the array is shorter\n * than this then integers are repeatedly used in a wrap-around fashion.\n */\n\n synchronized public void setSeed(int[] array)\n {\n if (array.length == 0)\n throw new IllegalArgumentException(\"Array length must be greater than zero\");\n int i, j, k;\n setSeed(19650218);\n i = 1;\n j = 0;\n k = (N > array.length ? N : array.length);\n for (; k != 0; k--) {\n mt[i] = (mt[i] ^ ((mt[i - 1] ^ (mt[i - 1] >>> 30)) * 1664525)) + array[j] + j; /*\n * non\n * linear\n */\n // mt[i] &= 0xffffffff; /* for WORDSIZE > 32 machines */\n i++;\n j++;\n if (i >= N) {\n mt[0] = mt[N - 1];\n i = 1;\n }\n if (j >= array.length)\n j = 0;\n }\n for (k = N - 1; k != 0; k--) {\n mt[i] = (mt[i] ^ ((mt[i - 1] ^ (mt[i - 1] >>> 30)) * 1566083941)) - i; /* non linear */\n // mt[i] &= 0xffffffff; /* for WORDSIZE > 32 machines */\n i++;\n if (i >= N) {\n mt[0] = mt[N - 1];\n i = 1;\n }\n }\n mt[0] = 0x80000000; /* MSB is 1; assuring non-zero initial array */\n }\n\n /**\n * Returns an integer with bits bits filled with a random number.\n */\n synchronized protected int next(int bits)\n {\n int y;\n\n if (mti >= N) // generate N words at one time\n {\n int kk;\n final int[] mt = this.mt; // locals are slightly faster\n final int[] mag01 = this.mag01; // locals are slightly faster\n\n for (kk = 0; kk < N - M; kk++) {\n y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);\n mt[kk] = mt[kk + M] ^ (y >>> 1) ^ mag01[y & 0x1];\n }\n for (; kk < N - 1; kk++) {\n y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK);\n mt[kk] = mt[kk + (M - N)] ^ (y >>> 1) ^ mag01[y & 0x1];\n }\n y = (mt[N - 1] & UPPER_MASK) | (mt[0] & LOWER_MASK);\n mt[N - 1] = mt[M - 1] ^ (y >>> 1) ^ mag01[y & 0x1];\n\n mti = 0;\n }\n\n y = mt[mti++];\n y ^= y >>> 11; // TEMPERING_SHIFT_U(y)\n y ^= (y << 7) & TEMPERING_MASK_B; // TEMPERING_SHIFT_S(y)\n y ^= (y << 15) & TEMPERING_MASK_C; // TEMPERING_SHIFT_T(y)\n y ^= (y >>> 18); // TEMPERING_SHIFT_L(y)\n\n return y >>> (32 - bits); // hope that's right!\n }\n\n /*\n * If you've got a truly old version of Java, you can omit these two next methods.\n */\n\n private synchronized void writeObject(ObjectOutputStream out)\n throws IOException\n {\n // just so we're synchronized.\n out.defaultWriteObject();\n }\n\n private synchronized void readObject(ObjectInputStream in)\n throws IOException, ClassNotFoundException\n {\n // just so we're synchronized.\n in.defaultReadObject();\n }\n\n /**\n * This method is missing from jdk 1.0.x and below. JDK 1.1 includes this for us, but what the\n * heck.\n */\n public boolean nextBoolean()\n {\n return next(1) != 0;\n }\n\n /**\n * This generates a coin flip with a probability probability of returning true, else\n * returning false. probability must be between 0.0 and 1.0, inclusive. Not as precise\n * a random real event as nextBoolean(double), but twice as fast. To explicitly use this,\n * remember you may need to cast to float first.\n */\n\n public boolean nextBoolean(float probability)\n {\n if (probability < 0.0f || probability > 1.0f)\n throw new IllegalArgumentException(\"probability must be between 0.0 and 1.0 inclusive.\");\n if (probability == 0.0f)\n return false; // fix half-open issues\n else if (probability == 1.0f)\n return true; // fix half-open issues\n return nextFloat() < probability;\n }\n\n /**\n * This generates a coin flip with a probability probability of returning true, else\n * returning false. probability must be between 0.0 and 1.0, inclusive.\n */\n\n public boolean nextBoolean(double probability)\n {\n if (probability < 0.0 || probability > 1.0)\n throw new IllegalArgumentException(\"probability must be between 0.0 and 1.0 inclusive.\");\n if (probability == 0.0)\n return false; // fix half-open issues\n else if (probability == 1.0)\n return true; // fix half-open issues\n return nextDouble() < probability;\n }\n\n /**\n * This method is missing from JDK 1.1 and below. JDK 1.2 includes this for us, but what the\n * heck.\n */\n\n public int nextInt(int n)\n {\n if (n <= 0)\n throw new IllegalArgumentException(\"n must be positive, got: \" + n);\n\n if ((n & -n) == n)\n return (int) ((n * (long) next(31)) >> 31);\n\n int bits, val;\n do {\n bits = next(31);\n val = bits % n;\n }\n while (bits - val + (n - 1) < 0);\n return val;\n }\n\n /**\n * This method is for completness' sake. Returns a long drawn uniformly from 0 to n-1. Suffice\n * it to say, n must be > 0, or an IllegalArgumentException is raised.\n */\n\n public long nextLong(long n)\n {\n if (n <= 0)\n throw new IllegalArgumentException(\"n must be positive, got: \" + n);\n\n long bits, val;\n do {\n bits = (nextLong() >>> 1);\n val = bits % n;\n }\n while (bits - val + (n - 1) < 0);\n return val;\n }\n\n /**\n * A bug fix for versions of JDK 1.1 and below. JDK 1.2 fixes this for us, but what the heck.\n */\n public double nextDouble()\n {\n return (((long) next(26) << 27) + next(27)) / (double) (1L << 53);\n }\n\n /**\n * Returns a double in the range from 0.0 to 1.0, possibly inclusive of 0.0 and 1.0 themselves.\n * Thus:\n * \n *
\n *
\n * | \n * | Expression\n * | Interval\n * | \n * | nextDouble(false, false)\n * | (0.0, 1.0)\n * |
\n * | nextDouble(true, false)\n * | [0.0, 1.0)\n * |
\n * | nextDouble(false, true)\n * | (0.0, 1.0]\n * |
\n * | nextDouble(true, true)\n * | [0.0, 1.0]\n * |
\n * \n * \n * This version preserves all possible random values in the double range.\n */\n public double nextDouble(boolean includeZero, boolean includeOne)\n {\n double d = 0.0;\n do {\n d = nextDouble(); // grab a value, initially from half-open [0.0, 1.0)\n if (includeOne && nextBoolean())\n d += 1.0; // if includeOne, with 1/2 probability, push to [1.0, 2.0)\n }\n while ((d > 1.0) || // everything above 1.0 is always invalid\n (!includeZero && d == 0.0)); // if we're not including zero, 0.0 is invalid\n return d;\n }\n\n /**\n * A bug fix for versions of JDK 1.1 and below. JDK 1.2 fixes this for us, but what the heck.\n */\n\n public float nextFloat()\n {\n return next(24) / ((float) (1 << 24));\n }\n\n /**\n * Returns a float in the range from 0.0f to 1.0f, possibly inclusive of 0.0f and 1.0f\n * themselves. Thus:\n * \n *
\n *
\n * | \n * | Expression\n * | Interval\n * | \n * | nextFloat(false, false)\n * | (0.0f, 1.0f)\n * |
\n * | nextFloat(true, false)\n * | [0.0f, 1.0f)\n * |
\n * | nextFloat(false, true)\n * | (0.0f, 1.0f]\n * |
\n * | nextFloat(true, true)\n * | [0.0f, 1.0f]\n * |
\n * \n * \n * This version preserves all possible random values in the float range.\n */\n public float nextFloat(boolean includeZero, boolean includeOne)\n {\n float d = 0.0f;\n do {\n d = nextFloat(); // grab a value, initially from half-open [0.0f, 1.0f)\n if (includeOne && nextBoolean())\n d += 1.0f; // if includeOne, with 1/2 probability, push to [1.0f, 2.0f)\n }\n while ((d > 1.0f) || // everything above 1.0f is always invalid\n (!includeZero && d == 0.0f)); // if we're not including zero, 0.0f is invalid\n return d;\n }\n\n /**\n * A bug fix for all versions of the JDK. The JDK appears to use all four bytes in an integer as\n * independent byte values! Totally wrong. I've submitted a bug report.\n */\n\n public void nextBytes(byte[] bytes)\n {\n for (int x = 0; x < bytes.length; x++)\n bytes[x] = (byte) next(8);\n }\n\n /** For completeness' sake, though it's not in java.util.Random. */\n\n public char nextChar()\n {\n // chars are 16-bit UniCode values\n return (char) (next(16));\n }\n\n /** For completeness' sake, though it's not in java.util.Random. */\n\n public short nextShort()\n {\n return (short) (next(16));\n }\n\n /** For completeness' sake, though it's not in java.util.Random. */\n\n public byte nextByte()\n {\n return (byte) (next(8));\n }\n\n /**\n * A bug fix for all JDK code including 1.2. nextGaussian can theoretically ask for the log of 0\n * and divide it by 0! See Java bug \n * http://developer.java.sun.com/developer/bugParade/bugs/4254501.html\n */\n\n synchronized public double nextGaussian()\n {\n if (__haveNextNextGaussian) {\n __haveNextNextGaussian = false;\n return __nextNextGaussian;\n }\n else {\n double v1, v2, s;\n do {\n v1 = 2 * nextDouble() - 1; // between -1.0 and 1.0\n v2 = 2 * nextDouble() - 1; // between -1.0 and 1.0\n s = v1 * v1 + v2 * v2;\n }\n while (s >= 1 || s == 0);\n double multiplier = StrictMath.sqrt(-2 * StrictMath.log(s) / s);\n __nextNextGaussian = v2 * multiplier;\n __haveNextNextGaussian = true;\n return v1 * multiplier;\n }\n }\n\n /**\n * Tests the code.\n */\n public static void main(String args[])\n {\n int j;\n\n MersenneTwister r;\n\n // CORRECTNESS TEST\n // COMPARE WITH http://www.math.keio.ac.jp/matumoto/CODES/MT2002/mt19937ar.out\n\n r = new MersenneTwister(new int[] { 0x123, 0x234, 0x345, 0x456 });\n System.out.println(\"Output of MersenneTwister with new (2002/1/26) seeding mechanism\");\n for (j = 0; j < 1000; j++) {\n // first, convert the int from signed to \"unsigned\"\n long l = (long) r.nextInt();\n if (l < 0)\n l += 4294967296L; // max int value\n String s = String.valueOf(l);\n while (s.length() < 10)\n s = \" \" + s; // buffer\n System.out.print(s + \" \");\n if (j % 5 == 4)\n System.out.println();\n }\n\n // SPEED TEST\n\n final long SEED = 4357;\n\n int xx;\n long ms;\n System.out.println(\"\\nTime to test grabbing 100000000 ints\");\n\n r = new MersenneTwister(SEED);\n ms = System.currentTimeMillis();\n xx = 0;\n for (j = 0; j < 100000000; j++)\n xx += r.nextInt();\n System.out.println(\"Mersenne Twister: \" + (System.currentTimeMillis() - ms)\n + \" Ignore this: \" + xx);\n\n System.out\n .println(\"To compare this with java.util.Random, run this same test on MersenneTwisterFast.\");\n System.out\n .println(\"The comparison with Random is removed from MersenneTwister because it is a proper\");\n System.out\n .println(\"subclass of Random and this unfairly makes some of Random's methods un-inlinable,\");\n System.out.println(\"so it would make Random look worse than it is.\");\n\n // TEST TO COMPARE TYPE CONVERSION BETWEEN\n // MersenneTwisterFast.java AND MersenneTwister.java\n\n System.out.println(\"\\nGrab the first 1000 booleans\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextBoolean() + \" \");\n if (j % 8 == 7)\n System.out.println();\n }\n if (!(j % 8 == 7))\n System.out.println();\n\n System.out\n .println(\"\\nGrab 1000 booleans of increasing probability using nextBoolean(double)\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextBoolean((double) (j / 999.0)) + \" \");\n if (j % 8 == 7)\n System.out.println();\n }\n if (!(j % 8 == 7))\n System.out.println();\n\n System.out\n .println(\"\\nGrab 1000 booleans of increasing probability using nextBoolean(float)\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextBoolean((float) (j / 999.0f)) + \" \");\n if (j % 8 == 7)\n System.out.println();\n }\n if (!(j % 8 == 7))\n System.out.println();\n\n byte[] bytes = new byte[1000];\n System.out.println(\"\\nGrab the first 1000 bytes using nextBytes\");\n r = new MersenneTwister(SEED);\n r.nextBytes(bytes);\n for (j = 0; j < 1000; j++) {\n System.out.print(bytes[j] + \" \");\n if (j % 16 == 15)\n System.out.println();\n }\n if (!(j % 16 == 15))\n System.out.println();\n\n byte b;\n System.out.println(\"\\nGrab the first 1000 bytes -- must be same as nextBytes\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print((b = r.nextByte()) + \" \");\n if (b != bytes[j])\n System.out.print(\"BAD \");\n if (j % 16 == 15)\n System.out.println();\n }\n if (!(j % 16 == 15))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 shorts\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextShort() + \" \");\n if (j % 8 == 7)\n System.out.println();\n }\n if (!(j % 8 == 7))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 ints\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextInt() + \" \");\n if (j % 4 == 3)\n System.out.println();\n }\n if (!(j % 4 == 3))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 ints of different sizes\");\n r = new MersenneTwister(SEED);\n int max = 1;\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextInt(max) + \" \");\n max *= 2;\n if (max <= 0)\n max = 1;\n if (j % 4 == 3)\n System.out.println();\n }\n if (!(j % 4 == 3))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 longs\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextLong() + \" \");\n if (j % 3 == 2)\n System.out.println();\n }\n if (!(j % 3 == 2))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 longs of different sizes\");\n r = new MersenneTwister(SEED);\n long max2 = 1;\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextLong(max2) + \" \");\n max2 *= 2;\n if (max2 <= 0)\n max2 = 1;\n if (j % 4 == 3)\n System.out.println();\n }\n if (!(j % 4 == 3))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 floats\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextFloat() + \" \");\n if (j % 4 == 3)\n System.out.println();\n }\n if (!(j % 4 == 3))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 doubles\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextDouble() + \" \");\n if (j % 3 == 2)\n System.out.println();\n }\n if (!(j % 3 == 2))\n System.out.println();\n\n System.out.println(\"\\nGrab the first 1000 gaussian doubles\");\n r = new MersenneTwister(SEED);\n for (j = 0; j < 1000; j++) {\n System.out.print(r.nextGaussian() + \" \");\n if (j % 3 == 2)\n System.out.println();\n }\n if (!(j % 3 == 2))\n System.out.println();\n\n }\n\n}\n"
},
{
"path": "src/utility/Parallel.java",
"content": "package utility;\n\nimport java.util.Collection;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ForkJoinPool;\nimport java.util.concurrent.RecursiveAction;\nimport java.util.concurrent.RecursiveTask;\n\n/**\n * Utilities for parallel computing in loops over independent tasks. This class\n * provides convenient methods for parallel processing of tasks that involve\n * loops over indices, in which computations for different indices are\n * independent.\n * \n * As a simple example, consider the following function that squares floats in\n * one array and stores the results in a second array.\n * \n *
\n * \n * static void sqr(float[] a, float[] b) {\n * int n = a.length;\n * for (int i=0; i<n; ++i)\n * b[i] = a[i]*a[i];\n * }\n * \n *
\n * \n * A serial version of a similar function for 2D arrays is:\n * \n * \n * \n * static void sqrSerial(float[][] a, float[][] b) \n * {\n * int n = a.length;\n * for (int i=0; i<n; ++i) {\n * sqr(a[i],b[i]);\n * }\n * \n *
\n * \n * Using this class, the parallel version for 2D arrays is:\n * \n * \n * \n * static void sqrParallel(final float[][] a, final float[][] b) {\n * int n = a.length;\n * Parallel.loop(n,new Parallel.LoopInt() {\n * public void compute(int i) {\n * sqr(a[i],b[i]);\n * }\n * });\n * }\n * \n *
\n * \n * In the parallel version, the method {@code compute} defined by the interface\n * {@code LoopInt} will be called n times for different indices i in the range\n * [0,n-1]. The order of indices is both indeterminant and irrelevant because\n * the computation for each index i is independent. The arrays a and b are\n * declared final as required for use in the implementation of {@code LoopInt}.\n * \n * Note: because the method {@code loop} and interface {@code LoopInt} are\n * static members of this class, we can omit the class name prefix\n * {@code Parallel} if we first import these names with\n * \n *
\n * \n * import static edu.mines.jtk.util.Parallel.*;\n * \n *
\n * \n * A similar method facilitates tasks that reduce a sequence of indexed values\n * to one or more values. For example, given the following method:\n * \n * \n * \n * static float sum(float[] a) {\n * int n = a.length;\n * float s = 0.0f;\n * for (int i=0; i<n; ++i)\n * s += a[i];\n * return s;\n * }\n * \n *
\n * \n * serial and parallel versions for 2D arrays may be written as:\n * \n * \n * \n * static float sumSerial(float[][] a) {\n * int n = a.length;\n * float s = 0.0f;\n * for (int i=0; i<n; ++i)\n * s += sum(a[i]);\n * return s;\n * }\n * \n *
\n * \n * and\n * \n * \n * \n * static float sumParallel(final float[][] a) {\n * int n = a.length;\n * return Parallel.reduce(n,new Parallel.ReduceInt<Float>() {\n * public Float compute(int i) {\n * return sum(a[i]);\n * }\n * public Float combine(Float s1, Float s2) {\n * return s1+s2;\n * }\n * });\n * }\n * \n *
\n * \n * In the parallel version, we implement the interface {@code ReduceInt} with\n * two methods, one to {@code compute} sums of array elements and another to\n * {@code combine} two such sums together. The same pattern works for other\n * reduce operations. For example, with similar functions we could compute\n * minimum and maximum values (in a single reduce) for any indexed sequence of\n * values.\n * \n * More general loops are supported, and are equivalent to the following serial\n * code:\n * \n *
\n * \n * for (int i=begin; i<end; i+=step)\n * // some computation that depends on i\n * \n *
\n * \n * The methods loop and reduce require that begin is less than end and that step\n * is positive. The requirement that begin is less than end ensures that reduce\n * is always well-defined. The requirement that step is positive ensures that\n * the loop terminates.\n * \n * Static methods loop and reduce submit tasks to a fork-join framework that\n * maintains a pool of threads shared by all users of these methods. These\n * methods recursively split tasks so that disjoint sets of indices are\n * processed in parallel by different threads.\n *
\n * In addition to the three loop parameters begin, end, and step, a fourth\n * parameter chunk may be specified. This chunk parameter is a threshold for\n * splitting tasks so that they can be performed in parallel. If a range of\n * indices to be processed is smaller than the chunk size, or if too many tasks\n * have already been queued for processing, then the indices are processed\n * serially. Otherwise, the range is split into two parts for processing by new\n * tasks. If specified, the chunk size is a lower bound; the number of indices\n * processed serially will never be lower, but may be higher, than a specified\n * chunk size. The default chunk size is one.\n *
\n * The default chunk size is often sufficient, because the test for an excess\n * number of queued tasks prevents tasks from being split needlessly. This test\n * is especially useful when parallel loops are nested, as when looping over\n * elements of multi-dimensional arrays.\n *
\n * For example, an implementation of the method {@code sqrParallel} for 3D\n * arrays could simply call the 2D version listed above. Tasks will naturally\n * tend to be split for outer loops, but not inner loops, thereby reducing\n * overhead, time spent splitting and queueing tasks.\n *
\n * Reference: A Java Fork/Join Framework, by Doug Lea, describes the framework\n * used to implement this class. This framework will be part of JDK 7.\n * \n * @author Dave Hale, Colorado School of Mines\n * @version 2010.11.23\n */\npublic class Parallel\n{\n\n\t/** A loop body that computes something for an int index. */\n\tpublic interface LoopInt\n\t{\n\n\t\t/**\n\t\t * Computes for the specified loop index.\n\t\t * \n\t\t * @param i\n\t\t * loop index.\n\t\t */\n\t\tpublic void compute(int i);\n\t}\n\n\t/** A loop body that computes and returns a value for an int index. */\n\tpublic interface ReduceInt\n\t{\n\n\t\t/**\n\t\t * Returns a value computed for the specified loop index.\n\t\t * \n\t\t * @param i\n\t\t * loop index.\n\t\t * @return the computed value.\n\t\t */\n\t\tpublic V compute(int i);\n\n\t\t/**\n\t\t * Returns the combination of two specified values.\n\t\t * \n\t\t * @param v1\n\t\t * a value.\n\t\t * @param v2\n\t\t * a value.\n\t\t * @return the combined value.\n\t\t */\n\t\tpublic V combine(V v1, V v2);\n\t}\n\n\t/**\n\t * A wrapper for objects that are not thread-safe. Such objects have methods\n\t * that cannot safely be executed concurrently in multiple threads. To use\n\t * an unsafe object within a parallel computation, first construct an\n\t * instance of this wrapper. Then, within the compute method, get the unsafe\n\t * object; if null, construct and set a new unsafe object in this wrapper,\n\t * before using the unsafe object to perform the computation. This pattern\n\t * ensures that each thread computes using a distinct unsafe object. For\n\t * example,\n\t * \n\t * \n\t * \n\t * final Parallel.Unsafe<Worker> nts = new Parallel.Unsafe<Worker>();\n\t * Parallel.loop(count,new Parallel.LoopInt() {\n\t * public void compute(int i) {\n\t * Worker w = nts.get(); // get worker for the current thread\n\t * if (w==null) nts.set(w=new Worker()); // if null, make one\n\t * w.work(); // the method work need not be thread-safe\n\t * }\n\t * });\n\t * \n\t *
\n\t * \n\t * This wrapper is most useful when (1) the cost of constructing an unsafe\n\t * object is high, relative to the cost of each call to compute, and (2) the\n\t * number of threads calling compute is significantly lower than the total\n\t * number of such calls. Otherwise, if either of these conditions is false,\n\t * then simply construct a new unsafe object within the compute method.\n\t * \n\t * This wrapper works much like the Java standard class ThreadLocal, except\n\t * that an object within this wrapper can be garbage-collected before its\n\t * thread dies. This difference is important because fork-join worker\n\t * threads are pooled and will typically die only when a program ends.\n\t */\n\tpublic static class Unsafe\n\t{\n\n\t\t/**\n\t\t * Constructs a wrapper for objects that are not thread-safe.\n\t\t */\n\t\tpublic Unsafe()\n\t\t{\n\t\t\tint initialCapacity = 16; // the default initial capacity\n\t\t\tfloat loadFactor = 0.5f; // huge numbers of threads are unlikely\n\t\t\tint concurrencyLevel = 2 * _pool.getParallelism();\n\t\t\t_map = new ConcurrentHashMap(initialCapacity,\n\t\t\t\tloadFactor, concurrencyLevel);\n\t\t}\n\n\t\t/**\n\t\t * Gets the object in this wrapper for the current thread.\n\t\t * \n\t\t * @return the object; null, of not yet set for the current thread.\n\t\t */\n\t\tpublic T get()\n\t\t{\n\t\t\treturn _map.get(Thread.currentThread());\n\t\t}\n\n\t\t/**\n\t\t * Sets the object in this wrapper for the current thread.\n\t\t * \n\t\t * @param object\n\t\t * the object.\n\t\t */\n\t\tpublic void set(T object)\n\t\t{\n\t\t\t_map.put(Thread.currentThread(), object);\n\t\t}\n\n\t\t/**\n\t\t * Returns a collection of all unsafe objects in this wrapper. This\n\t\t * method is useful only after parallel loops have ended.\n\t\t * \n\t\t * @return the collection of unsafe objects.\n\t\t */\n\t\tpublic Collection getAll()\n\t\t{\n\t\t\treturn _map.values();\n\t\t}\n\n\t\tprivate final ConcurrentHashMap _map;\n\t}\n\n\t/**\n\t * Performs a loop for (int i=0; i<end; ++i).\n\t * \n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param body\n\t * the loop body.\n\t */\n\tpublic static void loop(int end, LoopInt body)\n\t{\n\t\tloop(0, end, 1, 1, body);\n\t}\n\n\t/**\n\t * Performs a loop for (int i=begin; i<end; ++i).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param body\n\t * the loop body.\n\t */\n\tpublic static void loop(int begin, int end, LoopInt body)\n\t{\n\t\tloop(begin, end, 1, 1, body);\n\t}\n\n\t/**\n\t * Performs a loop for (int i=begin; i<end; i+=step).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param step\n\t * the index increment; must be positive.\n\t * @param body\n\t * the loop body.\n\t */\n\tpublic static void loop(int begin, int end, int step, LoopInt body)\n\t{\n\t\tloop(begin, end, step, 1, body);\n\t}\n\n\t/**\n\t * Performs a loop for (int i=begin; i<end; i+=step).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param step\n\t * the index increment; must be positive.\n\t * @param chunk\n\t * the chunk size; must be positive.\n\t * @param body\n\t * the loop body.\n\t */\n\tpublic static void loop(int begin, int end, int step, int chunk,\n\t\tLoopInt body)\n\t{\n\t\tcheckArgs(begin, end, step, chunk);\n\t\tif (_serial || end <= begin + chunk * step) {\n\t\t\tfor (int i = begin; i < end; i += step) {\n\t\t\t\tbody.compute(i);\n\t\t\t}\n\t\t}\n\t\telse {\n\t\t\tLoopIntAction task = new LoopIntAction(begin, end, step, chunk,\n\t\t\t\tbody);\n\t\t\tif (LoopIntAction.inForkJoinPool()) {\n\t\t\t\ttask.invoke();\n\t\t\t}\n\t\t\telse {\n\t\t\t\t_pool.invoke(task);\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Performs a reduce for (int i=0; i<end; ++i).\n\t * \n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param body\n\t * the loop body.\n\t * @return the computed value.\n\t */\n\tpublic static V reduce(int end, ReduceInt body)\n\t{\n\t\treturn reduce(0, end, 1, 1, body);\n\t}\n\n\t/**\n\t * Performs a reduce for (int i=begin; i<end; ++i).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param body\n\t * the loop body.\n\t * @return the computed value.\n\t */\n\tpublic static V reduce(int begin, int end, ReduceInt body)\n\t{\n\t\treturn reduce(begin, end, 1, 1, body);\n\t}\n\n\t/**\n\t * Performs a reduce for (int i=begin; i<end; i+=step).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param step\n\t * the index increment; must be positive.\n\t * @param body\n\t * the loop body.\n\t * @return the computed value.\n\t */\n\tpublic static V reduce(int begin, int end, int step, ReduceInt body)\n\t{\n\t\treturn reduce(begin, end, step, 1, body);\n\t}\n\n\t/**\n\t * Performs a reduce for (int i=begin; i<end; i+=step).\n\t * \n\t * @param begin\n\t * the begin index for the loop; must be less than end.\n\t * @param end\n\t * the end index (not included) for the loop.\n\t * @param step\n\t * the index increment; must be positive.\n\t * @param chunk\n\t * the chunk size; must be positive.\n\t * @param body\n\t * the loop body.\n\t * @return the computed value.\n\t */\n\tpublic static V reduce(int begin, int end, int step, int chunk,\n\t\tReduceInt body)\n\t{\n\t\tcheckArgs(begin, end, step, chunk);\n\t\tif (_serial || end <= begin + chunk * step) {\n\t\t\tV v = body.compute(begin);\n\t\t\tfor (int i = begin + step; i < end; i += step) {\n\t\t\t\tV vi = body.compute(i);\n\t\t\t\tv = body.combine(v, vi);\n\t\t\t}\n\t\t\treturn v;\n\t\t}\n\t\telse {\n\t\t\tReduceIntTask task = new ReduceIntTask(begin, end, step,\n\t\t\t\tchunk, body);\n\t\t\tif (ReduceIntTask.inForkJoinPool()) {\n\t\t\t\treturn task.invoke();\n\t\t\t}\n\t\t\telse {\n\t\t\t\treturn _pool.invoke(task);\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Enables or disables parallel processing by all methods of this class. By\n\t * default, parallel processing is enabled. If disabled, all tasks will be\n\t * executed on the current thread.\n\t * \n\t * Setting this flag to false disables parallel processing for all\n\t * users of this class. This method should therefore be used for\n\t * testing and benchmarking only.\n\t * \n\t * @param parallel\n\t * true, for parallel processing; false, otherwise.\n\t */\n\tpublic static void setParallel(boolean parallel)\n\t{\n\t\t_serial = !parallel;\n\t}\n\n\t// /////////////////////////////////////////////////////////////////////////\n\t// private\n\n\t// Implementation notes:\n\t// Each fork-join task below has a range of indices to be processed.\n\t// If the range is less than or equal to the chunk size, or if the\n\t// queue for the current thread holds too many tasks already, then\n\t// simply process the range on the current thread. Otherwise, split\n\t// the range into two parts that are approximately equal, ensuring\n\t// that the left part is at least as large as the right part. If the\n\t// right part is not empty, fork a new task. Then compute the left\n\t// part in the current thread, and, if necessary, join the right part.\n\n\t// Threshold for number of surplus queued tasks. Used below to\n\t// determine whether or not to split a task into two subtasks.\n\tprivate static final int NSQT = 6;\n\n\t// The pool shared by all fork-join tasks created through this class.\n\tprivate static ForkJoinPool _pool = new ForkJoinPool();\n\n\t// Serial flag; true for no parallel processing.\n\tprivate static boolean _serial = false;\n\n\t/**\n\t * Checks loop arguments.\n\t */\n\tprivate static void checkArgs(int begin, int end, int step, int chunk)\n\t{\n\t\targument(begin < end, \"begin 0, \"step>0\");\n\t\targument(chunk > 0, \"chunk>0\");\n\t}\n\n\tpublic static void argument(boolean condition, String message)\n\t{\n\t\tif (!condition)\n\t\t\tthrow new IllegalArgumentException(\"required condition: \" + message);\n\t}\n\n\t/**\n\t * Splits range [begin:end) into [begin:middle) and [middle:end). The\n\t * returned middle index equals begin plus an integer multiple of step.\n\t */\n\tprivate static int middle(int begin, int end, int step)\n\t{\n\t\treturn begin + step + ((end - begin - 1) / 2) / step * step;\n\t}\n\n\t/**\n\t * Fork-join task for parallel loop.\n\t */\n\tprivate static class LoopIntAction\n\t\textends RecursiveAction\n\t{\n\t\tLoopIntAction(int begin, int end, int step, int chunk, LoopInt body)\n\t\t{\n\t\t\tassert begin < end : \"begin < end\";\n\t\t\t_begin = begin;\n\t\t\t_end = end;\n\t\t\t_step = step;\n\t\t\t_chunk = chunk;\n\t\t\t_body = body;\n\t\t}\n\n\t\t@Override\n\t\tprotected void compute()\n\t\t{\n\t\t\tif (_end <= _begin + _chunk * _step\n\t\t\t\t|| getSurplusQueuedTaskCount() > NSQT) {\n\t\t\t\tfor (int i = _begin; i < _end; i += _step) {\n\t\t\t\t\t_body.compute(i);\n\t\t\t\t}\n\t\t\t}\n\t\t\telse {\n\t\t\t\tint middle = middle(_begin, _end, _step);\n\t\t\t\tLoopIntAction l = new LoopIntAction(_begin, middle, _step,\n\t\t\t\t\t_chunk, _body);\n\t\t\t\tLoopIntAction r = (middle < _end) ? new LoopIntAction(middle,\n\t\t\t\t\t_end, _step, _chunk, _body) : null;\n\t\t\t\tif (r != null)\n\t\t\t\t\tr.fork();\n\t\t\t\tl.compute();\n\t\t\t\tif (r != null)\n\t\t\t\t\tr.join();\n\t\t\t}\n\t\t}\n\n\t\tprivate final int _begin, _end, _step, _chunk;\n\t\tprivate final LoopInt _body;\n\t}\n\n\t/**\n\t * Fork-join task for parallel reduce.\n\t */\n\tprivate static class ReduceIntTask\n\t\textends RecursiveTask\n\t{\n\t\tReduceIntTask(int begin, int end, int step, int chunk, ReduceInt body)\n\t\t{\n\t\t\tassert begin < end : \"begin < end\";\n\t\t\t_begin = begin;\n\t\t\t_end = end;\n\t\t\t_step = step;\n\t\t\t_chunk = chunk;\n\t\t\t_body = body;\n\t\t}\n\n\t\t@Override\n\t\tprotected V compute()\n\t\t{\n\t\t\tif (_end <= _begin + _chunk * _step\n\t\t\t\t|| getSurplusQueuedTaskCount() > NSQT) {\n\t\t\t\tV v = _body.compute(_begin);\n\t\t\t\tfor (int i = _begin + _step; i < _end; i += _step) {\n\t\t\t\t\tV vi = _body.compute(i);\n\t\t\t\t\tv = _body.combine(v, vi);\n\t\t\t\t}\n\t\t\t\treturn v;\n\t\t\t}\n\t\t\telse {\n\t\t\t\tint middle = middle(_begin, _end, _step);\n\t\t\t\tReduceIntTask l = new ReduceIntTask(_begin, middle,\n\t\t\t\t\t_step, _chunk, _body);\n\t\t\t\tReduceIntTask r = (middle < _end) ? new ReduceIntTask(\n\t\t\t\t\tmiddle, _end, _step, _chunk, _body) : null;\n\t\t\t\tif (r != null)\n\t\t\t\t\tr.fork();\n\t\t\t\tV v = l.compute();\n\t\t\t\tif (r != null)\n\t\t\t\t\tv = _body.combine(v, r.join());\n\t\t\t\treturn v;\n\t\t\t}\n\t\t}\n\n\t\tprivate final int _begin, _end, _step, _chunk;\n\t\tprivate final ReduceInt _body;\n\t}\n}"
},
{
"path": "test/corpus.LABEL",
"content": "apple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\napple\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\ngoogle\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\nmicrosoft\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\ntwitter\n"
},
{
"path": "test/corpus.txt",
"content": "iphone crack iphone \nadding support iphone announced \nyoutube video guy siri pretty love \nrim made easy switch iphone yeah \nrealized ios \ncurrent blackberry user bit disappointed move android iphone \nthings siri sooo glad gave siri sense humor \ngreat personal event tonight store \ncompanies experience customer service \napply job hope call lol \nlmao siri find hide body \nregistered developer appreciated \nwow great deals ipad gen offers great deals gen ipads \nlearning trip hong kong gotta hand iphones apps \ndark side hey send free iphone publicly burn blackberry \nfind mac air \nmacbook keyboard lunch break today warranty \nipads replace \nsiri amazing \namazing ios feature \nreply featured education apps website today sweet \nreply useless days \niphone yesterday awesome amount info \nquestion brother iphone \npeople iphone phone happy \nceo points ios \nbus iphone \number appstore itunes mobile devices talking desktop application \nbring ipad ipad set red red ipad \nsells million iphone weekend steve jobs lives iphone \napologize \ndownloads ios users \nlmfao argument siri \nincredible million iphone screenshot days iphone iphone \nfixed ios battery drain problem replacement iphone working \nbrand macbook professional macbook years miss time \nsiri dad mom brother girlfriend \nstore amazing call waiting music \nsweet replaced \nbad sells million iphones debut weekend smartphone \nloving technology iphone mac air icloud technology \nloving ios update \nmention store great customer service store \ntime iphone forward man longer paying texts \ngirlfriend iphone great \nicloud set works cloud \nmommy totally email company great service store \nloving ios upgrade iphone \nios ipad \nmaking switch android iphone iphone smartphone store \nincredible people offering water macbook professional wow \nmacbook sick \nplay man loving camera iphone facebook \nyeah ios changed life \nreader worldwide web \nlove service case hand case \nyears jobs iphone iphone iphone \nblackberry years lost service moving iphone \nsells million iphones days \nweekend iphone \nmacbook professional year time selling android \npost card \nputting kind glad hear alive \ngod youtube bad ass system loving \ndays iphone nice gave \nios email lock screen opening unlocking \nword wow iphone weekend sales top million \nlove ios easter eggs pull middle top bottom pulls awesome feature ios \nlove ios easter eggs pull middle top bottom pulls awesome feature \nrun beautiful morning man love ios iphone \nsimply \nmade happy text lol text \nday great customer service received today phone phone \nloving ipod update \nupgraded iphone siri worth upgrade forward siri \ngreat world missed \nloving iphone ios \nlove \niphone great genius \ncards application card arrived local post office today \niphone siri \nmeet siri iphone click link \nwork feel worst \nios upgrade good luck blackberry \nloving ios awesome \niphone addicted club \nguy playing facetime watching game bar \nblackberry boo powered technology work \nipod time iphone good job guys \nsiri year lead lost \nios sweet notifications phone search covers mail wifi sync icloud backup integrated \ngreat james story today times retail success \nworld due ios guys \nimpressive service genius bar metro center power replaced free screen replacement free \nnice guy store replaced phone showed crack screen \niphone battery longer day happened edge iphone nice job \nminutes write blackberry showing \neye phone impressed \niphone space amazing products people things \nmaking ipad feel ios \nnexus good feel bit guess android users android \nnice game helps search \nnice game helps search facebook \nbuild website website free \nandroid ics pretty good worth \nandroid ice cream sandwich nexus android nexus \nexciting day ice cream sandwich day android \nwow nexus beautiful totally gonna market share smart phone market \nintegrated data usage manager brilliant design watching lol \nice cream sandwich android works htc desire \nice cream sandwich sounds android ice cream sandwich \namazing imo android missing \nforget phone nice feature android nexus \nfinally unveiled android ice cream sandwich good \nfinally searches logged users \nrim strategy released hours release ics \nman love galaxy nexus samsung android \ndoubt \nshare winning war \ndear galaxy nexus send email technology \ntelegraph reports biggest threat facebook power users \nsamsung made bad android king \nfacebook power users telegraph socialmedia \nimpressed android update good font design \nvideo wallet wow \ntweet remember spell straight \nandroid samsung nexus \nefficient fun releases infinite digital bookcase \npass social seo facebook \nice cream sandwich stop carriers bullying smartphone users android \nagree freaking awesome \nicecream great \nhelps \nsamsung galaxy nexus iphone \nice cream sandwich delicious iphone launches android aka \nloving \nsamsung push mobile experience forward \nfinally power volume screenshot ics \nnexus press conference slick \nhigh school appreciated \nscream scream scream android job major game mobile space \nthinking ahead \nventurebeat virtual bookcase sharing \nandroid phone keeping iphone \nandroid ice cream sandwich feature closer roboto type face read \nwork samsung android ics impressive \nadd profile webgl project add addthis \nwork company work \ninvention \nwait ice cream sandwich android \nstop nexus \nphone \nandroid device updated galaxy nexus \nandroid introducing ice cream sandwich delicious version android ics \nexcited android features android ics \nwait nexus play \ncheck video introducing galaxy nexus simple beautiful smart youtube android nexus \ncream ice cream phone job \ngreat small businesses platform features thoughts \nloves presentations tool docs adding video \nbrilliant webgl bookcase \nsearches things \nandroid ice cream introducing galaxy nexus simple beautiful smart \nnexus prime android \ninteresting bookcase venturebeat releases infinite digital bookcase \ngood finally focus user experience android \nics awesome phone android motorola \niphone ice cream sandwich android \nnexus line smart move \nandroid beam alright made team team android \nandroid reply font good start ics \nice cream sandwich face unlock works \nready ice cream sandwich ics nexus android android \nice cream sandwich android \ntaste ice cream sandwich bite \nsamsung event live blog gadget haven android \nandroid ice cream sandwich make smartphone operating systems \nphoto sharing people application ice cream sandwich imo ics \nandroid nexus phone makes iphone cheap store android \nsweet ice cream sandwich android ice cream sandwich officially ics \nraise hand android powered phone samsung \nsiri android device replace iphone \nnexus page live nexus android \nexcited android beam face unlock android ics \nlinkedin tools company page contact \nsamsung ice cream sandwich samsung \nintroducing galaxy nexus simple beautiful smart android ics samsung \nglad design android shows waiting \nthoughts android ics excited play features android \nregister galaxy nexus android \nwow webgl infinite bookcase \nics awesome wait face unlock android \ngotta pretty android chrome android \nnovember direct purchase samsung \nnexus wanna awesome \nevent time change android samsung \nios user ics awesome great job \nyeah great job ics \nliterally mind blown samsung \nmotorola verizon perfect \nopens door spanish entrepreneurs project \nintel ibm \nwindows phone mango update process ahead schedule mango \nback smartphone rich \nword works computer \nfree gen stores \nwatch codename data explorer ctp coming \nlunch today vslive \nwatch codename data explorer ctp coming month \ndetails search improvements windows start screen \nmango shows taste smartphone success mango \nawesome moving dev finally local \nstores offer free windows phone devices \nstores offer free windows phone devices neowin \nstore spend hard vslive \nfree west check \nhey parents free tools kids online live family \ncloud offers students free access improve tcn \nawesome bit \ndetails windows search improvements \nyeah taking metro yeah good android \nlove kids tech \nexplains improvements windows start screen search tech \nsearch idea search great \nbing king search search \npowerpoint users power create service bye solutions \nfuture information innovators nov info \ncurate personal history project greenwich month \nbeam research project \ngreat sql server session \nworks days \nballmer thinks computer scientist android tech agree \ngreat time \nwin server works fine vmware \nwow tech turns body touchscreen psfk \nlove love feeling building vslive bringing conference \nresearch shows awesome step closer bit kinect \nresearch shows science science fact cool sound \nresearch shows science science fact \nzune music canada music news \nkinect makes learning playful education \nmango \ncheck change world \ngood world wait \nwatching windows pretty impressive finally mac interesting battle store \nxbox share \ngod \nblog post cool tool mouse tools \nforget siri beating speech commands mango siri \ntests proves appsense enterprise capability users personalization database enterprise \nsoftware good points sap dynamics \ngood dev \nsecure anti \nimpressed creating images \nmac blown marketing \nyahoo sale years back bought glad deal year \nomnitouch impressive technology \ngood bing paying \nipads windows tablets study \nhome day great time \nmango shows taste smartphone success \npicture services cloud love \nwindows net dev \nnice talk community \nomg sharepoint working \ninnovation sad sad \noffice love genius \nlove gates foundation \ngood \nskype family amazing things \nabsolutely loving mouse \nfan cool video turn surface touchscreen \nwow android ics lots talk mango launch people public speaking \nupdated computer windows \nics android kill mango nokia \npeople names mail week \noutlook mac sucks hate \nxbox accounts hack reports \nupdate net \nwindows media center fail \neclipsed \nword upgrade doc doc word won open doc suck \nu.s. antitrust leaving business played dumb \nlync crash issue mac fixed \nbroke played engages racketeering calls respect \nnokia chief executive mole \nfrozen xbox live xbl accounts online games report hacked \ngave windows dev preview good waiting beta windows \npowerpoint fix powerpoint presentations \neclipsed guardian \nkind search \ngreat time family advertising \nwindows forget past antitrust issues \npaying make racketeering \nday talking talk tomorrow waiting \nreader compares albatross neck agree join \nlot word freeze minutes \nlol perfect simple hate windows phones \nmonths months lose \nreader compares albatross neck agree join discussion \nmake sleep plan \nfeel world put facebook blackberry helps \nmiss boo \neverytime leave back back telling lol \napplication ass theme \nsleep sleep \nstarting sending hashtags emails taking lives \nshit lol hell \ntoday introduced social media love \nfacebook \nyeah shows glad \npretty facebook \ngotta love shit round world speed \nbed gonna minute bed \ndear fucking missed today internet \ntweet keeping busy school \ngood thing people left social \nsocial media \nguess addicted university exam questions \ngood thing people left social side \napples facebook content \nbed favorite application facebook \nfacebook change makes excited privacy \nimpressive numbers smm socialmedia \nfuck facebook bullshit bitch \ncool love \nfuck facebook follow \nhaven shit man haven fun \nfind song end television show watched \nliterally back facebook text email technology good \nisn pretty damn amazing hope year fast \ndear missed promise touch \nbored sad mad happy true friend \nfacebook sucks amp \nshit funny haven shit day \nvoice people real life lol \nyeah time bug \nscience hashtags facebook \nfeeling real world \nbiggest \nfacebook messed make add reliable \nfreaking kidding wth \ntomorrow blue ass bird continued \ndead \nemails telling \nsucks follow \npeople reporting retweets working technical problem \nback lol \nretweets broken haven tuesday \ntomorrow blue ass bird ass \nain showing current mentions tweets \ngonna problems fixed asap \nretweets \nman boring \napplication show touch tweet \ntrouble application updating application \nmessed everytime text message \nshow fucking retweets bitch \nsooo trash \nshowing retweets shit \nmom argument pretty \naddicted care \nappreciated start working computer \nretweets section account working hours problem \ngood send bloody tweets \nfeel \nmake account \nfucking late damn \ndear fix shit retweets mentions \ndead fuck \npoint \ngiving tweets tweeted past days lol \nmessed followers numbers \ntimeline mentions shit \ngarbage \nhell television man \nstupid fucking give damn mentions ugh \nfucking \nfacebook television wanna study \nshow retweets ill back facebook \nreply opinions \nforget day time haven \nblogs tumblr \ntalk step game \nreminder fail \njoin follow \nways competition \npeople facebook day life \ndrop follow show love \ntelling reply \nsleep time \nemotions \ncall night \nwork break time yeah \ntumblr love \nage year days hours minutes seconds find \nwanna aye shit living \nshout favorite people happy girls \nfollow back \nsleep good people trip \n"
},
{
"path": "test/corpus_test.txt",
"content": "making ipad feel ios \nnexus good feel bit guess android users android \nnice game helps search \nnice game helps search facebook \nbuild website website free \nandroid ics pretty good worth \nandroid ice cream sandwich nexus android nexus \nexciting day ice cream sandwich day android \nwow nexus beautiful totally gonna market share smart phone market \nintegrated data usage manager brilliant design watching lol \nice cream sandwich android works htc desire \nice cream sandwich sounds android ice cream sandwich \namazing imo android missing \nforget phone nice feature android nexus \nfinally unveiled android ice cream sandwich good \nfinally searches logged users \nrim strategy released hours release ics \nman love galaxy nexus samsung android \ndoubt \nshare winning war \ndear galaxy nexus send email technology \ntelegraph reports biggest threat facebook power users \nsamsung made bad android king \nfacebook power users telegraph socialmedia \nimpressed android update good font design \nvideo wallet wow \ntweet remember spell straight \nandroid samsung nexus \nefficient fun releases infinite digital bookcase \npass social seo facebook \nice cream sandwich stop carriers bullying smartphone users android \nagree freaking awesome \nicecream great \nhelps \nsamsung galaxy nexus iphone \nice cream sandwich delicious iphone launches android aka \nloving \nsamsung push mobile experience forward \nfinally power volume screenshot ics \nnexus press conference slick \nhigh school appreciated \nscream scream scream android job major game mobile space \nthinking ahead \nventurebeat virtual bookcase sharing \nandroid phone keeping iphone \nandroid ice cream sandwich feature closer roboto type face read \nwork samsung android ics impressive \nadd profile webgl project add addthis \nwork company work \ninvention \nwait ice cream sandwich android \nstop nexus \nphone \nandroid device updated galaxy nexus \nandroid introducing ice cream sandwich delicious version android ics \nexcited android features android ics \nwait nexus play \ncheck video introducing galaxy nexus simple beautiful smart youtube android nexus \ncream ice cream phone job \ngreat small businesses platform features thoughts \nloves presentations tool docs adding video \nbrilliant webgl bookcase \nsearches things \nandroid ice cream introducing galaxy nexus simple beautiful smart \nnexus prime android \ninteresting bookcase venturebeat releases infinite digital bookcase \ngood finally focus user experience android \nics awesome phone android motorola \niphone ice cream sandwich android \nnexus line smart move \nandroid beam alright made team team android \nandroid reply font good start ics \nice cream sandwich face unlock works \nready ice cream sandwich ics nexus android android \nice cream sandwich android \ntaste ice cream sandwich bite \nsamsung event live blog gadget haven android \nandroid ice cream sandwich make smartphone operating systems \nphoto sharing people application ice cream sandwich imo ics \nandroid nexus phone makes iphone cheap store android \nsweet ice cream sandwich android ice cream sandwich officially ics \nraise hand android powered phone samsung \nsiri android device replace iphone \nnexus page live nexus android \nexcited android beam face unlock android ics \nlinkedin tools company page contact \nsamsung ice cream sandwich samsung \nintroducing galaxy nexus simple beautiful smart android ics samsung \nglad design android shows waiting \nthoughts android ics excited play features android \nregister galaxy nexus android \nwow webgl infinite bookcase \nics awesome wait face unlock android \ngotta pretty android chrome android \nnovember direct purchase samsung \nnexus wanna awesome \nevent time change android samsung \nios user ics awesome great job \nyeah great job ics \nliterally mind blown samsung \nmotorola verizon perfect \nopens door spanish entrepreneurs project \nintel ibm \nwindows phone mango update process ahead schedule mango \nback smartphone rich \nword works computer \nfree gen stores \nwatch codename data explorer ctp coming \nlunch today vslive \nwatch codename data explorer ctp coming month \ndetails search improvements windows start screen \nmango shows taste smartphone success mango \nawesome moving dev finally local \nstores offer free windows phone devices \nstores offer free windows phone devices neowin \nstore spend hard vslive \nfree west check \nhey parents free tools kids online live family \ncloud offers students free access improve tcn \nawesome bit \ndetails windows search improvements \nyeah taking metro yeah good android \nlove kids tech \nexplains improvements windows start screen search tech \nsearch idea search great \nbing king search search \npowerpoint users power create service bye solutions \nfuture information innovators nov info \ncurate personal history project greenwich month \nbeam research project \ngreat sql server session \nworks days \nballmer thinks computer scientist android tech agree \ngreat time \nwin server works fine vmware \nwow tech turns body touchscreen psfk \nlove love feeling building vslive bringing conference \nresearch shows awesome step closer bit kinect \nresearch shows science science fact cool sound \nresearch shows science science fact \nzune music canada music news \nkinect makes learning playful education \nmango \ncheck change world \ngood world wait \nwatching windows pretty impressive finally mac interesting battle store \nxbox share \ngod \nblog post cool tool mouse tools \nforget siri beating speech commands mango siri \ntests proves appsense enterprise capability users personalization database enterprise \nsoftware good points sap dynamics \ngood dev \nsecure anti \nimpressed creating images \nmac blown marketing \nyahoo sale years back bought glad deal year \nomnitouch impressive technology \ngood bing paying \nipads windows tablets study \nhome day great time \nmango shows taste smartphone success \npicture services cloud love \nwindows net dev \nnice talk community \nomg sharepoint working \ninnovation sad sad \noffice love genius \nlove gates foundation \ngood \nskype family amazing things \nabsolutely loving mouse \nfan cool video turn surface touchscreen \nwow android ics lots talk mango launch people public speaking \nupdated computer windows \nics android kill mango nokia \npeople names mail week \noutlook mac sucks hate \nxbox accounts hack reports \nupdate net \nwindows media center fail \neclipsed \nword upgrade doc doc word won open doc suck \nu.s. antitrust leaving business played dumb \nlync crash issue mac fixed \nbroke played engages racketeering calls respect \nnokia chief executive mole \nfrozen xbox live xbl accounts online games report hacked \ngave windows dev preview good waiting beta windows \npowerpoint fix powerpoint presentations \neclipsed guardian \nkind search \ngreat time family advertising \nwindows forget past antitrust issues \npaying make racketeering \nday talking talk tomorrow waiting \nreader compares albatross neck agree join \nlot word freeze minutes \nlol perfect simple hate windows phones \nmonths months lose \n"
}
]