[
  {
    "path": ".gitignore",
    "content": "*.class\n\n# Mobile Tools for Java (J2ME)\n.mtj.tmp/\n\n# Package Files #\n*.jar\n*.war\n*.ear\n\n# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml\nhs_err_pid*\n\n.idea/\n*.iml\ntarget/\n"
  },
  {
    "path": ".travis.yml",
    "content": "# Enable container-based infrastructure\nsudo: false\nlanguage: java\ninstall: mvn install -DskipTests -Dgpg.skip\njdk:\n- oraclejdk8\n"
  },
  {
    "path": "LICENSE",
    "content": "Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"{}\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright {yyyy} {name of copyright owner}\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "README.adoc",
    "content": "= Time series compression library, based on the Facebook's Gorilla paper\n:source-language: java\n\nifdef::env-github[]\n[link=https://travis-ci.org/burmanm/gorilla-tsc]\nimage::https://travis-ci.org/burmanm/gorilla-tsc.svg?branch=master[Build Status,70,18]\n[link=https://maven-badges.herokuapp.com/maven-central/fi.iki.yak/compression-gorilla]\nimage::https://img.shields.io/maven-central/v/fi.iki.yak/compression-gorilla.svg[Maven central]\nendif::[]\n\n== Introduction\n\nThis is Java based implementation of the compression methods described in the paper link:http://www.vldb.org/pvldb/vol8/p1816-teller.pdf[\"Gorilla: A Fast, Scalable, In-Memory Time Series Database\"]. For explanation on how the compression methods work, read the excellent paper.\n\nIn comparison to the original paper, this implementation allows using both integer values (`long`) as well as\nfloating point values (`double`), both 64 bit in length.\n\nVersions 1.x and 2.x are not compatible with each other due to small differences to the stored array. Versions 2.x\nwill support reading and storing older format also, see usage for more details.\n\n== Usage\n\nThe included tests are a good source for examples.\n\n=== Maven\n\n[source, xml]\n----\n    <dependency>\n        <groupId>fi.iki.yak</groupId>\n        <artifactId>compression-gorilla</artifactId>\n    </dependency>\n----\n\nYou can find latest version from the maven logo link above.\n\n=== Compressing\n\nTo compress in the older 1.x format, use class ``Compressor``. For 2.x, use ``GorillaCompressor`` (recommended).\n``LongArrayOutput`` is also recommended compared to ``ByteBufferBitOutput`` because of performance. One can supply\nalternative predictor to the ``GorillaCompressor`` if required. One such implementation is included,\n``DifferentialFCM`` that provides better compression ratio for some data patterns.\n\n[source, java]\n----\nlong now = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.HOURS)\n        .toInstant(ZoneOffset.UTC).toEpochMilli();\n\nLongArrayOutput output = new LongArrayOutput();\nGorillaCompressor c = new GorillaCompressor(now, output);\n----\n\nCompression class requires a block timestamp and an implementation of `BitOutput` interface.\n\n[source, java]\n----\nc.addValue(long, double);\n----\n\nAdds a new floating-point value to the time series. If you wish to store only long values, use `c.addValue(long,\nlong)`, however do `not` mix these in the same series.\n\nAfter the block is ready, remember to call:\n\n[source, java]\n----\nc.close();\n----\n\nwhich flushes the remaining data to the stream and writes closing information.\n\n=== Decompressing\n\nTo decompress from the older 1.x format, use class ``Decompressor``. For 2.x, use ``GorillaDecompressor`` (recommended).\n``LongArrayInput`` is also recommended compared to ``ByteBufferBitInput`` because of performance if the 2.x\nformat was used to compress the time series. If the original compressor used different predictor than\n``LastValuePredictor`` it must be defined in the constructor.\n\n[source, java]\n----\nLongArrayInput input = new LongArrayInput(byteBuffer);\nGorillaDecompressor d = new GorillaDecompressor(input);\n----\n\nTo decompress a stream of bytes, supply `GorillaDecompressor` with a suitable implementation of `BitInput` interface.\n The LongArrayInput allows to decompress a long array or existing `ByteBuffer` presentation with 8 byte word\n length.\n\n[source, java]\n----\nPair pair = d.readPair();\n----\n\nRequesting next pair with `readPair()` returns the following series value or a `null` once the series is completely\nread. The pair is a simple placeholder object with `getTimestamp()` and `getDoubleValue()` or `getLongValue()`.\n\n== Performance\n\nThe following performance in reached in a Linux VM running on VMware Player in Windows 8.1 host. i7 2600K at 4GHz.\nThe benchmark used is the ``EncodingBenchmark``. These results should not be directly compared to other\nimplementations unless similar dataset is used.\n\nResults are in millions of datapoints (timestamp + value) pairs per second. The values in this benchmark are\nin doubles (performance with longs is slightly higher, around ~2-3M/s).\n\n.Compression\n|===\n|GorillaCompressor (2.0.0) |Compressor (1.1.0)\n\n|83.5M/s (~1.34GB/s)\n|31.2M/s (~499MB/s)\n|===\n\n\n.Decompression\n|===\n|GorillaDecompressor (2.0.0) |Decompressor (1.1.0)\n\n|77,9M/s (~1.25GB/s)\n|51.4M/s (~822MB/s)\n|===\n\nMost of the differences in decompression / compression speed between versions come from implementation changes and\nnot from the small changes to the output format.\n\n== Roadmap\n\nThere were few things I wanted to get to 2.0.0, but had to decide against due to lack of time. I will implement these\n later with potentially some breaking API changes:\n\n * Support timestamp only compressions (2.2.x)\n * Include ByteBufferLongOutput/ByteBufferLongInput in the package (2.2.x)\n * Move bit operations to inside the GorillaCompressor/GorillaDecompressor to allow easier usage with\n other allocators (2.2.x)\n\n== Internals\n\n=== Differences to the original paper\n\n* Maximum number of leadingZeros is stored with 6 bits to allow up to 63 leading zeros, which are necessary when\nstoring long values. (>= 2.0.0)\n* Timestamp delta-of-delta are stored by first turning them with ZigZag encoding to positive integers and then\nreduced by one to fit in the necessary bits. In the decoding phase all the values are incremented by one to fetch the\n original value. (>= 2.0.0)\n* The compressed blocks are created with a 27 bit delta header (unlike in the original paper, which uses a 14 bit delta\n  header). This allows to use up to one day block size using millisecond precision. (>= 1.0.0)\n\n=== Data structure\n\nValues must be inserted in the increasing time order, out-of-order insertions are not supported.\n\nThe included ByteBufferBitInput and ByteBufferBitOutput classes use a big endian order for the data.\n\n== Contributing\n\nFile an issue and/or send a pull request.\n\n=== License\n\n....\n   Copyright 2016-2018 Michael Burman and/or other contributors.\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n....\n\n"
  },
  {
    "path": "pom.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd\">\n    <modelVersion>4.0.0</modelVersion>\n\n    <groupId>fi.iki.yak</groupId>\n    <artifactId>compression-gorilla</artifactId>\n    <version>2.1.2-SNAPSHOT</version>\n    <name>Gorilla time series compression in Java</name>\n    <description>Implements the time series compression methods as described in the Facebook's Gorilla\n        paper</description>\n    <url>https://github.com/burmanm/gorilla-tsc</url>\n    <licenses>\n        <license>\n            <name>Apache License, Version 2.0</name>\n            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>\n            <distribution>repo</distribution>\n        </license>\n    </licenses>\n\n    <properties>\n        <java.version>1.8</java.version>\n        <junit.jupiter.version>5.0.0-M4</junit.jupiter.version>\n        <junit.platform.version>1.0.0-M4</junit.platform.version>\n        <!--<junit.vintage.version>4.12.0-M2</junit.vintage.version>-->\n        <jmh.version>1.18</jmh.version>\n        <jar.name>benchmark</jar.name>\n        <maven.release.plugin.version>2.5.3</maven.release.plugin.version>\n    </properties>\n\n    <scm>\n        <url>https://github.com/burmanm/gorilla-tsc</url>\n        <connection>scm:git:git://github.com/burmam/gorilla-tsc.git</connection>\n        <developerConnection>scm:git:git@github.com:burmanm/gorilla-tsc.git</developerConnection>\n        <tag>HEAD</tag>\n    </scm>\n\n    <developers>\n        <developer>\n            <email>yak@iki.fi</email>\n            <name>Michael Burman</name>\n            <url>https://github.com/burmanm</url>\n            <id>burmanm</id>\n        </developer>\n    </developers>\n\n    <distributionManagement>\n        <repository>\n            <id>ossrh</id>\n            <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>\n        </repository>\n    </distributionManagement>\n\n    <dependencies>\n        <dependency>\n            <groupId>org.junit.jupiter</groupId>\n            <artifactId>junit-jupiter-engine</artifactId>\n            <version>${junit.jupiter.version}</version>\n            <scope>test</scope>\n        </dependency>\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-core</artifactId>\n            <version>${jmh.version}</version>\n        </dependency>\n        <dependency>\n            <groupId>org.openjdk.jmh</groupId>\n            <artifactId>jmh-generator-annprocess</artifactId>\n            <version>${jmh.version}</version>\n            <scope>provided</scope>\n        </dependency>\n    </dependencies>\n\n    <build>\n        <plugins>\n            <plugin>\n                <artifactId>maven-compiler-plugin</artifactId>\n                <version>3.1</version>\n                <configuration>\n                    <source>${java.version}</source>\n                    <target>${java.version}</target>\n                </configuration>\n            </plugin>\n            <plugin>\n                <artifactId>maven-surefire-plugin</artifactId>\n                <version>2.19</version>\n                <dependencies>\n                    <dependency>\n                        <groupId>org.junit.platform</groupId>\n                        <artifactId>junit-platform-surefire-provider</artifactId>\n                        <version>${junit.platform.version}</version>\n                    </dependency>\n                </dependencies>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-shade-plugin</artifactId>\n                <version>2.4.3</version>\n                <executions>\n                    <execution>\n                        <phase>package</phase>\n                        <goals>\n                            <goal>shade</goal>\n                        </goals>\n                        <configuration>\n                            <finalName>${jar.name}</finalName>\n                            <transformers>\n                                <transformer implementation=\"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer\">\n                                    <mainClass>org.openjdk.jmh.Main</mainClass>\n                                </transformer>\n                            </transformers>\n                        </configuration>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-release-plugin</artifactId>\n                <version>${maven.release.plugin.version}</version>\n                <configuration>\n                    <autoVersionSubmodules>true</autoVersionSubmodules>\n                    <useReleaseProfile>false</useReleaseProfile>\n                    <releaseProfiles>release</releaseProfiles>\n                    <goals>deploy</goals>\n                </configuration>\n            </plugin>\n            <plugin>\n                <groupId>org.sonatype.plugins</groupId>\n                <artifactId>nexus-staging-maven-plugin</artifactId>\n                <version>1.6.7</version>\n                <extensions>true</extensions>\n                <configuration>\n                    <serverId>ossrh</serverId>\n                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>\n                    <autoReleaseAfterClose>false</autoReleaseAfterClose>\n                </configuration>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-javadoc-plugin</artifactId>\n                <executions>\n                    <execution>\n                        <goals>\n                            <goal>jar</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-source-plugin</artifactId>\n                <executions>\n                    <execution>\n                        <goals>\n                            <goal>jar</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n            <plugin>\n                <groupId>org.apache.maven.plugins</groupId>\n                <artifactId>maven-gpg-plugin</artifactId>\n                <version>1.5</version>\n                <executions>\n                    <execution>\n                        <id>sign-artifacts</id>\n                        <phase>verify</phase>\n                        <goals>\n                            <goal>sign</goal>\n                        </goals>\n                    </execution>\n                </executions>\n            </plugin>\n        </plugins>\n    </build>\n</project>"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/BitInput.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\n/**\n * This interface is used for reading a compressed time series.\n *\n * @author Michael Burman\n */\npublic interface BitInput {\n\n    /**\n     * Reads the next bit and returns true if bit is set and false if not.\n     *\n     * @return true == 1, false == 0\n     */\n    boolean readBit();\n\n    /**\n     * Returns a long that was stored in the next X bits in the stream.\n     *\n     * @param bits Amount of least significant bits to read from the stream.\n     * @return reads the next long in the series using bits meaningful bits\n     */\n    long getLong(int bits);\n\n    /**\n     * Read until next unset bit is found, or until maxBits has been reached.\n     *\n     * @param maxBits How many bits at maximum until returning\n     * @return Integer value of the read bits\n     */\n    int nextClearBit(int maxBits);\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/BitOutput.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\n/**\n * This interface is used to write a compressed timeseries.\n *\n * @author Michael Burman\n */\npublic interface BitOutput {\n\n    /**\n     * Stores a single bit and increases the bitcount by 1\n     */\n    void writeBit();\n\n    /**\n     * Stores a 0 and increases the bitcount by 1\n     */\n    void skipBit();\n\n    /**\n     * Write the given long value using the defined amount of least significant bits.\n     *\n     * @param value The long value to be written\n     * @param bits How many bits are stored to the stream\n     */\n    void writeBits(long value, int bits);\n\n    /**\n     * Flushes the current byte to the underlying stream\n     */\n    void flush();\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/ByteBufferBitInput.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport java.nio.ByteBuffer;\n\n/**\n * An implementation of BitInput that parses the data from byte array or existing ByteBuffer.\n *\n * @author Michael Burman\n */\npublic class ByteBufferBitInput implements BitInput {\n    private ByteBuffer bb;\n    private byte b;\n    private int bitsLeft = 0;\n\n    /**\n     * Uses an existing ByteBuffer to read the stream. Starts at the ByteBuffer's current position.\n     *\n     * @param buf Use existing ByteBuffer\n     */\n    public ByteBufferBitInput(ByteBuffer buf) {\n        bb = buf;\n        flipByte();\n    }\n\n    public ByteBufferBitInput(byte[] input) {\n        this(ByteBuffer.wrap(input));\n    }\n\n    /**\n     * Reads the next bit and returns a boolean representing it.\n     *\n     * @return true if the next bit is 1, otherwise 0.\n     */\n    public boolean readBit() {\n        boolean bit = ((b >> (bitsLeft - 1)) & 1) == 1;\n        bitsLeft--;\n        flipByte();\n        return bit;\n    }\n\n    /**\n     * Reads a long from the next X bits that represent the least significant bits in the long value.\n     *\n     * @param bits How many next bits are read from the stream\n     * @return long value that was read from the stream\n     */\n    public long getLong(int bits) {\n        long value = 0;\n        while(bits > 0) {\n            if(bits > bitsLeft || bits == Byte.SIZE) {\n                // Take only the bitsLeft \"least significant\" bits\n                byte d = (byte) (b & ((1<<bitsLeft) - 1));\n                value = (value << bitsLeft) + (d & 0xFF);\n                bits -= bitsLeft;\n                bitsLeft = 0;\n            } else {\n                // Shift to correct position and take only least significant bits\n                byte d = (byte) ((b >>> (bitsLeft - bits)) & ((1<<bits) - 1));\n                value = (value << bits) + (d & 0xFF);\n                bitsLeft -= bits;\n                bits = 0;\n            }\n            flipByte();\n        }\n        return value;\n    }\n\n    @Override\n    public int nextClearBit(int maxBits) {\n        int val = 0x00;\n\n        for(int i = 0; i < maxBits; i++) {\n            val <<= 1;\n            boolean bit = readBit();\n\n            if(bit) {\n                val |= 0x01;\n            } else {\n                break;\n            }\n        }\n        return val;\n    }\n\n    private void flipByte() {\n        if (bitsLeft == 0) {\n            b = bb.get();\n            bitsLeft = Byte.SIZE;\n        }\n    }\n\n    /**\n     * Returns the underlying ByteBuffer\n     *\n     * @return ByteBuffer that's connected to the underlying stream\n     */\n    public ByteBuffer getByteBuffer() {\n        return this.bb;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/ByteBufferBitOutput.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport java.nio.ByteBuffer;\n\n/**\n * An implementation of BitOutput interface that uses off-heap storage.\n *\n * @author Michael Burman\n */\npublic class ByteBufferBitOutput implements BitOutput {\n    public static final int DEFAULT_ALLOCATION = 4096;\n\n    private ByteBuffer bb;\n    private byte b;\n    private int bitsLeft = Byte.SIZE;\n\n    /**\n     * Creates a new ByteBufferBitOutput with a default allocated size of 4096 bytes.\n     */\n    public ByteBufferBitOutput() {\n        this(DEFAULT_ALLOCATION);\n    }\n\n    /**\n     * Give an initialSize different than DEFAULT_ALLOCATIONS. Recommended to use values which are dividable by 4096.\n     *\n     * @param initialSize New initialsize to use\n     */\n    public ByteBufferBitOutput(int initialSize) {\n        bb = ByteBuffer.allocateDirect(initialSize);\n        b = bb.get(bb.position());\n    }\n\n    private void expandAllocation() {\n        ByteBuffer largerBB = ByteBuffer.allocateDirect(bb.capacity()*2);\n        bb.flip();\n        largerBB.put(bb);\n        largerBB.position(bb.capacity());\n        bb = largerBB;\n    }\n\n    private void flipByte() {\n        if(bitsLeft == 0) {\n            bb.put(b);\n            if(!bb.hasRemaining()) {\n                expandAllocation();\n            }\n            b = bb.get(bb.position());\n            bitsLeft = Byte.SIZE;\n        }\n    }\n\n    @Override\n    public void writeBit() {\n        b |= (1 << (bitsLeft - 1));\n        bitsLeft--;\n        flipByte();\n    }\n\n    @Override\n    public void skipBit() {\n        bitsLeft--;\n        flipByte();\n    }\n\n    /**\n     * Writes the given long to the stream using bits amount of meaningful bits.\n     *\n     * @param value Value to be written to the stream\n     * @param bits How many bits are stored to the stream\n     */\n    public void writeBits(long value, int bits) {\n        while(bits > 0) {\n            int shift = bits - bitsLeft;\n            if(shift >= 0) {\n                b |= (byte) ((value >> shift) & ((1 << bitsLeft) - 1));\n                bits -= bitsLeft;\n                bitsLeft = 0;\n            } else {\n                shift = bitsLeft - bits;\n                b |= (byte) (value << shift);\n                bitsLeft -= bits;\n                bits = 0;\n            }\n            flipByte();\n        }\n    }\n\n    /**\n     * Causes the currently handled byte to be written to the stream\n     */\n    @Override\n    public void flush() {\n        bitsLeft = 0;\n        flipByte(); // Causes write to the ByteBuffer\n    }\n\n    /**\n     * Returns the underlying DirectByteBuffer\n     *\n     * @return ByteBuffer of type DirectByteBuffer\n     */\n    public ByteBuffer getByteBuffer() {\n        return this.bb;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/Compressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\n/**\n * Implements the time series compression as described in the Facebook's Gorilla Paper. Value compression\n * is for floating points only.\n *\n * @author Michael Burman\n */\npublic class Compressor {\n\n    private int storedLeadingZeros = Integer.MAX_VALUE;\n    private int storedTrailingZeros = 0;\n    private long storedVal = 0;\n    private long storedTimestamp = 0;\n    private long storedDelta = 0;\n\n    private long blockTimestamp = 0;\n\n    public final static short FIRST_DELTA_BITS = 27;\n\n    private BitOutput out;\n\n    // We should have access to the series?\n    public Compressor(long timestamp, BitOutput output) {\n        blockTimestamp = timestamp;\n        out = output;\n        addHeader(timestamp);\n    }\n\n    private void addHeader(long timestamp) {\n        // One byte: length of the first delta\n        // One byte: precision of timestamps\n        out.writeBits(timestamp, 64);\n    }\n\n    /**\n     * Adds a new long value to the series. Note, values must be inserted in order.\n     *\n     * @param timestamp Timestamp which is inside the allowed time block (default 24 hours with millisecond precision)\n     * @param value next floating point value in the series\n     */\n    public void addValue(long timestamp, long value) {\n        if(storedTimestamp == 0) {\n            writeFirst(timestamp, value);\n        } else {\n            compressTimestamp(timestamp);\n            compressValue(value);\n        }\n    }\n\n    /**\n     * Adds a new double value to the series. Note, values must be inserted in order.\n     *\n     * @param timestamp Timestamp which is inside the allowed time block (default 24 hours with millisecond precision)\n     * @param value next floating point value in the series\n     */\n    public void addValue(long timestamp, double value) {\n        if(storedTimestamp == 0) {\n            writeFirst(timestamp, Double.doubleToRawLongBits(value));\n        } else {\n            compressTimestamp(timestamp);\n            compressValue(Double.doubleToRawLongBits(value));\n        }\n    }\n\n    private void writeFirst(long timestamp, long value) {\n        storedDelta = timestamp - blockTimestamp;\n        storedTimestamp = timestamp;\n        storedVal = value;\n\n        out.writeBits(storedDelta, FIRST_DELTA_BITS);\n        out.writeBits(storedVal, 64);\n    }\n\n    /**\n     * Closes the block and writes the remaining stuff to the BitOutput.\n     */\n    public void close() {\n        // These are selected to test interoperability and correctness of the solution, this can be read with go-tsz\n        out.writeBits(0x0F, 4);\n        out.writeBits(0xFFFFFFFF, 32);\n        out.skipBit();\n        out.flush();\n    }\n\n    /**\n     * Difference to the original Facebook paper, we store the first delta as 27 bits to allow\n     * millisecond accuracy for a one day block.\n     *\n     * Also, the timestamp delta-delta is not good for millisecond compressions..\n     *\n     * @param timestamp epoch\n     */\n    private void compressTimestamp(long timestamp) {\n        // a) Calculate the delta of delta\n        long newDelta = (timestamp - storedTimestamp);\n        long deltaD = newDelta - storedDelta;\n\n        // If delta is zero, write single 0 bit\n        if(deltaD == 0) {\n            out.skipBit();\n        } else if(deltaD >= -63 && deltaD <= 64) {\n            out.writeBits(0x02, 2); // store '10'\n            out.writeBits(deltaD, 7); // Using 7 bits, store the value..\n        } else if(deltaD >= -255 && deltaD <= 256) {\n            out.writeBits(0x06, 3); // store '110'\n            out.writeBits(deltaD, 9); // Use 9 bits\n        } else if(deltaD >= -2047 && deltaD <= 2048) {\n            out.writeBits(0x0E, 4); // store '1110'\n            out.writeBits(deltaD, 12); // Use 12 bits\n        } else {\n            out.writeBits(0x0F, 4); // Store '1111'\n            out.writeBits(deltaD, 32); // Store delta using 32 bits\n        }\n\n        storedDelta = newDelta;\n        storedTimestamp = timestamp;\n    }\n\n    private void compressValue(long value) {\n        // TODO Fix already compiled into a big method\n       long xor = storedVal ^ value;\n\n        if(xor == 0) {\n            // Write 0\n            out.skipBit();\n        } else {\n            int leadingZeros = Long.numberOfLeadingZeros(xor);\n            int trailingZeros = Long.numberOfTrailingZeros(xor);\n\n            // Check overflow of leading? Can't be 32!\n            if(leadingZeros >= 32) {\n                leadingZeros = 31;\n            }\n\n            // Store bit '1'\n            out.writeBit();\n\n            if(leadingZeros >= storedLeadingZeros && trailingZeros >= storedTrailingZeros) {\n                writeExistingLeading(xor);\n            } else {\n                writeNewLeading(xor, leadingZeros, trailingZeros);\n            }\n        }\n\n        storedVal = value;\n    }\n\n    /**\n     * If there at least as many leading zeros and as many trailing zeros as previous value, control bit = 0 (type a)\n     * store the meaningful XORed value\n     *\n     * @param xor XOR between previous value and current\n     */\n    private void writeExistingLeading(long xor) {\n        out.skipBit();\n        int significantBits = 64 - storedLeadingZeros - storedTrailingZeros;\n        out.writeBits(xor >>> storedTrailingZeros, significantBits);\n    }\n\n    /**\n     * store the length of the number of leading zeros in the next 5 bits\n     * store length of the meaningful XORed value in the next 6 bits,\n     * store the meaningful bits of the XORed value\n     * (type b)\n     *\n     * @param xor XOR between previous value and current\n     * @param leadingZeros New leading zeros\n     * @param trailingZeros New trailing zeros\n     */\n    private void writeNewLeading(long xor, int leadingZeros, int trailingZeros) {\n        out.writeBit();\n        out.writeBits(leadingZeros, 5); // Number of leading zeros in the next 5 bits\n\n        int significantBits = 64 - leadingZeros - trailingZeros;\n        out.writeBits(significantBits, 6); // Length of meaningful bits in the next 6 bits\n        out.writeBits(xor >>> trailingZeros, significantBits); // Store the meaningful bits of XOR\n\n        storedLeadingZeros = leadingZeros;\n        storedTrailingZeros = trailingZeros;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/Decompressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\n/**\n * Decompresses a compressed stream created by the Compressor. Returns pairs of timestamp and floating point value.\n *\n * @author Michael Burman\n */\npublic class Decompressor {\n\n    private int storedLeadingZeros = Integer.MAX_VALUE;\n    private int storedTrailingZeros = 0;\n    private long storedVal = 0;\n    private long storedTimestamp = 0;\n    private long storedDelta = 0;\n\n    private long blockTimestamp = 0;\n\n    private boolean endOfStream = false;\n\n    private BitInput in;\n\n    public Decompressor(BitInput input) {\n        in = input;\n        readHeader();\n    }\n\n    private void readHeader() {\n        blockTimestamp = in.getLong(64);\n    }\n\n    /**\n     * Returns the next pair in the time series, if available.\n     *\n     * @return Pair if there's next value, null if series is done.\n     */\n    public Pair readPair() {\n        next();\n        if(endOfStream) {\n            return null;\n        }\n        return new Pair(storedTimestamp, storedVal);\n    }\n\n    private void next() {\n        if (storedTimestamp == 0) {\n            // First item to read\n            storedDelta = in.getLong(Compressor.FIRST_DELTA_BITS);\n            if(storedDelta == (1<<27) - 1) {\n                endOfStream = true;\n                return;\n            }\n            storedVal = in.getLong(64);\n            storedTimestamp = blockTimestamp + storedDelta;\n        } else {\n            nextTimestamp();\n        }\n    }\n\n    private int bitsToRead() {\n        int val = in.nextClearBit(4);\n        int toRead = 0;\n\n        switch(val) {\n            case 0x00:\n                break;\n            case 0x02:\n                toRead = 7; // '10'\n                break;\n            case 0x06:\n                toRead = 9; // '110'\n                break;\n            case 0x0e:\n                toRead = 12;\n                break;\n            case 0x0F:\n                toRead = 32;\n                break;\n        }\n\n        return toRead;\n    }\n\n    private void nextTimestamp() {\n        // Next, read timestamp\n        long deltaDelta = 0;\n        int toRead = bitsToRead();\n        if (toRead > 0) {\n            deltaDelta = in.getLong(toRead);\n\n            if(toRead == 32) {\n                if ((int) deltaDelta == 0xFFFFFFFF) {\n                    // End of stream\n                    endOfStream = true;\n                    return;\n                }\n            } else {\n                // Turn \"unsigned\" long value back to signed one\n                if(deltaDelta > (1 << (toRead - 1))) {\n                    deltaDelta -= (1 << toRead);\n                }\n            }\n\n            deltaDelta = (int) deltaDelta;\n        }\n\n        storedDelta = storedDelta + deltaDelta;\n        storedTimestamp = storedDelta + storedTimestamp;\n        nextValue();\n    }\n\n    private void nextValue() {\n        // Read value\n        if (in.readBit()) {\n            // else -> same value as before\n            if (in.readBit()) {\n                // New leading and trailing zeros\n                storedLeadingZeros = (int) in.getLong(5);\n\n                byte significantBits = (byte) in.getLong(6);\n                if(significantBits == 0) {\n                    significantBits = 64;\n                }\n                storedTrailingZeros = 64 - significantBits - storedLeadingZeros;\n            }\n            long value = in.getLong(64 - storedLeadingZeros - storedTrailingZeros);\n            value <<= storedTrailingZeros;\n            value = storedVal ^ value;\n            storedVal = value;\n        }\n    }\n\n}"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaCompressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport fi.iki.yak.ts.compression.gorilla.predictors.LastValuePredictor;\n\n/**\n * Implements a slightly modified version of the time series compression as described in the Facebook's Gorilla\n * Paper.\n *\n * @author Michael Burman\n */\npublic class GorillaCompressor {\n\n    private long storedTimestamp = 0;\n    private int storedDelta = 0;\n\n    private long blockTimestamp = 0;\n\n    public final static int FIRST_DELTA_BITS = 27;\n\n    private static int DELTAD_7_MASK = 0x02 << 7;\n    private static int DELTAD_9_MASK = 0x06 << 9;\n    private static int DELTAD_12_MASK = 0x0E << 12;\n\n    private BitOutput out;\n\n    private ValueCompressor valueCompressor;\n\n    public GorillaCompressor(long timestamp, BitOutput output) {\n        this(timestamp, output, new LastValuePredictor());\n    }\n\n    public GorillaCompressor(long timestamp, BitOutput output, Predictor predictor) {\n        blockTimestamp = timestamp;\n        out = output;\n        addHeader(timestamp);\n        this.valueCompressor = new ValueCompressor(output, predictor);\n    }\n\n    private void addHeader(long timestamp) {\n        out.writeBits(timestamp, 64);\n    }\n\n    /**\n     * Adds a new long value to the series. Note, values must be inserted in order.\n     *\n     * @param timestamp Timestamp which is inside the allowed time block (default 24 hours with millisecond precision)\n     * @param value next floating point value in the series\n     */\n    public void addValue(long timestamp, long value) {\n        if(storedTimestamp == 0) {\n            writeFirst(timestamp, value);\n        } else {\n            compressTimestamp(timestamp);\n            valueCompressor.compressValue(value);\n        }\n    }\n\n    /**\n     * Adds a new double value to the series. Note, values must be inserted in order.\n     *\n     * @param timestamp Timestamp which is inside the allowed time block (default 24 hours with millisecond precision)\n     * @param value next floating point value in the series\n     */\n    public void addValue(long timestamp, double value) {\n        if(storedTimestamp == 0) {\n            writeFirst(timestamp, Double.doubleToRawLongBits(value));\n            return;\n        }\n        compressTimestamp(timestamp);\n        valueCompressor.compressValue(Double.doubleToRawLongBits(value));\n    }\n\n    private void writeFirst(long timestamp, long value) {\n        storedDelta = (int) (timestamp - blockTimestamp);\n        storedTimestamp = timestamp;\n\n        out.writeBits(storedDelta, FIRST_DELTA_BITS);\n        valueCompressor.writeFirst(value);\n    }\n\n    /**\n     * Closes the block and writes the remaining stuff to the BitOutput.\n     */\n    public void close() {\n        out.writeBits(0x0F, 4);\n        out.writeBits(0xFFFFFFFF, 32);\n        out.skipBit();\n        out.flush();\n    }\n\n    /**\n     * Difference to the original Facebook paper, we store the first delta as 27 bits to allow\n     * millisecond accuracy for a one day block.\n     *\n     * Also, the timestamp delta-delta is not good for millisecond compressions..\n     *\n     * @param timestamp epoch\n     */\n    private void compressTimestamp(long timestamp) {\n\n        // a) Calculate the delta of delta\n        int newDelta = (int) (timestamp - storedTimestamp);\n        int deltaD = newDelta - storedDelta;\n\n        if(deltaD == 0) {\n            out.skipBit();\n        } else {\n            deltaD = encodeZigZag32(deltaD);\n            deltaD--; // Increase by one in the decompressing phase as we have one free bit\n            int bitsRequired = 32 - Integer.numberOfLeadingZeros(deltaD); // Faster than highestSetBit\n\n            // Turns to inlineable tableswitch\n            switch(bitsRequired) {\n                case 1:\n                case 2:\n                case 3:\n                case 4:\n                case 5:\n                case 6:\n                case 7:\n                    deltaD |= DELTAD_7_MASK;\n                    out.writeBits(deltaD, 9);\n                    break;\n                case 8:\n                case 9:\n                    deltaD |= DELTAD_9_MASK;\n                    out.writeBits(deltaD, 12);\n                    break;\n                case 10:\n                case 11:\n                case 12:\n                    out.writeBits(deltaD | DELTAD_12_MASK, 16);\n                    break;\n                default:\n                    out.writeBits(0x0F, 4); // Store '1111'\n                    out.writeBits(deltaD, 32); // Store delta using 32 bits\n                    break;\n            }\n            storedDelta = newDelta;\n        }\n\n        storedTimestamp = timestamp;\n    }\n\n    // START: From protobuf\n\n    /**\n     * Encode a ZigZag-encoded 32-bit value.  ZigZag encodes signed integers\n     * into values that can be efficiently encoded with varint.  (Otherwise,\n     * negative values must be sign-extended to 64 bits to be varint encoded,\n     * thus always taking 10 bytes on the wire.)\n     *\n     * @param n A signed 32-bit integer.\n     * @return An unsigned 32-bit integer, stored in a signed int because\n     *         Java has no explicit unsigned support.\n     */\n    public static int encodeZigZag32(final int n) {\n        // Note:  the right-shift must be arithmetic\n        return (n << 1) ^ (n >> 31);\n    }\n\n    // END: From protobuf\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaDecompressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport java.util.stream.Stream;\n\nimport fi.iki.yak.ts.compression.gorilla.predictors.LastValuePredictor;\n\n/**\n * Decompresses a compressed stream created by the GorillaCompressor.\n *\n * @author Michael Burman\n */\npublic class GorillaDecompressor {\n    private long storedTimestamp = 0;\n    private long storedDelta = 0;\n\n    private long blockTimestamp = 0;\n    private long storedVal = 0;\n    private boolean endOfStream = false;\n\n    private final BitInput in;\n    private final ValueDecompressor decompressor;\n\n    public GorillaDecompressor(BitInput input) {\n        this(input, new LastValuePredictor());\n    }\n\n    public GorillaDecompressor(BitInput input, Predictor predictor) {\n        in = input;\n        readHeader();\n        this.decompressor = new ValueDecompressor(input, predictor);\n    }\n\n    private void readHeader() {\n        blockTimestamp = in.getLong(64);\n    }\n\n    /**\n     * Returns the next pair in the time series, if available.\n     *\n     * @return Pair if there's next value, null if series is done.\n     */\n    public Pair readPair() {\n        next();\n        if(endOfStream) {\n            return null;\n        }\n        Pair pair = new Pair(storedTimestamp, storedVal);\n        return pair;\n    }\n\n    private void next() {\n        // TODO I could implement a non-streaming solution also.. is there ever a need for streaming solution?\n\n        if(storedTimestamp == 0) {\n            first();\n            return;\n        }\n\n        nextTimestamp();\n    }\n\n    private void first() {\n        // First item to read\n        storedDelta = in.getLong(Compressor.FIRST_DELTA_BITS);\n        if(storedDelta == (1<<27) - 1) {\n            endOfStream = true;\n            return;\n        }\n        storedVal = decompressor.readFirst();\n//        storedVal = in.getLong(64);\n        storedTimestamp = blockTimestamp + storedDelta;\n    }\n\n    private void nextTimestamp() {\n        // Next, read timestamp\n        int readInstruction = in.nextClearBit(4);\n        long deltaDelta;\n\n        switch(readInstruction) {\n            case 0x00:\n                storedTimestamp = storedDelta + storedTimestamp;\n                storedVal = decompressor.nextValue();\n                return;\n            case 0x02:\n                deltaDelta = in.getLong(7);\n                break;\n            case 0x06:\n                deltaDelta = in.getLong(9);\n                break;\n            case 0x0e:\n                deltaDelta = in.getLong(12);\n                break;\n            case 0x0F:\n                deltaDelta = in.getLong(32);\n                // For storage save.. if this is the last available word, check if remaining bits are all 1\n                if ((int) deltaDelta == 0xFFFFFFFF) {\n                    // End of stream\n                    endOfStream = true;\n                    return;\n                }\n                break;\n            default:\n                return;\n        }\n\n        deltaDelta++;\n        deltaDelta = decodeZigZag32((int) deltaDelta);\n        storedDelta = storedDelta + deltaDelta;\n\n        storedTimestamp = storedDelta + storedTimestamp;\n        storedVal = decompressor.nextValue();\n    }\n\n    // START: From protobuf\n\n    /**\n     * Decode a ZigZag-encoded 32-bit value. ZigZag encodes signed integers into values that can be\n     * efficiently encoded with varint. (Otherwise, negative values must be sign-extended to 64 bits\n     * to be varint encoded, thus always taking 10 bytes on the wire.)\n     *\n     * @param n An unsigned 32-bit integer, stored in a signed int because Java has no explicit\n     *     unsigned support.\n     * @return A signed 32-bit integer.\n     */\n    public static int decodeZigZag32(final int n) {\n        return (n >>> 1) ^ -(n & 1);\n    }\n\n    // END: From protobuf\n}"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayInput.java",
    "content": "/*\n * Copyright 2016 Red Hat, Inc. and/or its affiliates\n * and other contributors as indicated by the @author tags.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage fi.iki.yak.ts.compression.gorilla;\n\n/**\n * Implements on-heap long array input stream\n *\n * @author Michael Burman\n */\npublic class LongArrayInput implements BitInput {\n    private final long[] longArray; // TODO Investigate also the ByteBuffer performance here.. or Unsafe\n    private long lB;\n    private int position = 0;\n    private int bitsLeft = 0;\n\n    public LongArrayInput(long[] array) {\n        this.longArray = array;\n        flipByte();\n    }\n\n    @Override\n    public boolean readBit() {\n        boolean bit = (lB & LongArrayOutput.BIT_SET_MASK[bitsLeft - 1]) != 0;\n        bitsLeft--;\n        checkAndFlipByte();\n        return bit;\n    }\n\n    private void flipByte() {\n        lB = longArray[position++];\n        bitsLeft = Long.SIZE;\n    }\n\n    private void checkAndFlipByte() {\n        if(bitsLeft == 0) {\n            flipByte();\n        }\n    }\n\n    @Override\n    public long getLong(int bits) {\n        long value;\n        if(bits <= bitsLeft) {\n            // We can read from this word only\n            // Shift to correct position and take only n least significant bits\n            value = (lB >>> (bitsLeft - bits)) & LongArrayOutput.MASK_ARRAY[bits - 1];\n            bitsLeft -= bits; // We ate n bits from it\n            checkAndFlipByte();\n        } else {\n            // This word and next one, no more (max bits is 64)\n            value = lB & LongArrayOutput.MASK_ARRAY[bitsLeft - 1]; // Read what's left first\n            bits -= bitsLeft;\n            flipByte(); // We need the next one\n            value <<= bits; // Give n bits of space to value\n            value |= (lB >>> (bitsLeft - bits));\n            bitsLeft -= bits;\n        }\n        return value;\n    }\n\n    @Override\n    public int nextClearBit(int maxBits) {\n        int val = 0x00;\n\n        for(int i = 0; i < maxBits; i++) {\n            val <<= 1;\n            // TODO This loop has too many branches and unnecessary boolean casts\n            boolean bit = readBit();\n\n            if(bit) {\n                val |= 0x01;\n            } else {\n                break;\n            }\n        }\n        return val;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayOutput.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport java.util.Arrays;\n\n/**\n * An implementation of BitOutput interface that uses on-heap long array.\n *\n * @author Michael Burman\n */\npublic class LongArrayOutput implements BitOutput {\n    public static final int DEFAULT_ALLOCATION = 256;\n\n    private long[] longArray;\n    private int position = 0;\n\n    protected long lB;\n    protected int bitsLeft = Long.SIZE;\n\n    public final static long[] MASK_ARRAY;\n    public final static long[] BIT_SET_MASK;\n\n    // Java does not allow creating 64 bit masks with (1L << 64) - 1; (end result is 0)\n    static {\n        MASK_ARRAY = new long[64];\n        long mask = 1;\n        long value = 0;\n        for (int i = 0; i < MASK_ARRAY.length; i++) {\n            value = value | mask;\n            mask = mask << 1;\n\n            MASK_ARRAY[i] = value;\n        }\n\n        BIT_SET_MASK = new long[64];\n        for(int i = 0; i < BIT_SET_MASK.length; i++) {\n            BIT_SET_MASK[i] = (1L << i);\n        }\n    }\n\n\n    /**\n     * Creates a new ByteBufferBitOutput with a default allocated size of 4096 bytes.\n     */\n    public LongArrayOutput() {\n        this(DEFAULT_ALLOCATION);\n    }\n\n    /**\n     * Give an initialSize different than DEFAULT_ALLOCATIONS. Recommended to use values which are dividable by 4096.\n     *\n     * @param initialSize New initialsize to use\n     */\n    public LongArrayOutput(int initialSize) {\n        longArray = new long[initialSize];\n        lB = longArray[position];\n    }\n\n    protected void expandAllocation() {\n        long[] largerArray = new long[longArray.length*2];\n        System.arraycopy(longArray, 0, largerArray, 0, longArray.length);\n        longArray = largerArray;\n    }\n\n    private void checkAndFlipByte() {\n        // Wish I could avoid this check in most cases...\n        if(bitsLeft == 0) {\n            flipWord();\n        }\n    }\n\n    protected int capacityLeft() {\n        return longArray.length - position;\n    }\n\n    protected void flipWord() {\n        if(capacityLeft() <= 2) { // We want to have always at least 2 longs available\n            expandAllocation();\n        }\n        flipWordWithoutExpandCheck();\n    }\n\n    protected void flipWordWithoutExpandCheck() {\n        longArray[position] = lB;\n        ++position;\n        resetInternalWord();\n    }\n\n    private void resetInternalWord() {\n        lB = 0;\n        bitsLeft = Long.SIZE;\n    }\n\n    /**\n     * Sets the next bit (or not) and moves the bit pointer.\n     */\n    public void writeBit() {\n        lB |= BIT_SET_MASK[bitsLeft - 1];\n        bitsLeft--;\n        checkAndFlipByte();\n    }\n\n    public void skipBit() {\n        bitsLeft--;\n        checkAndFlipByte();\n    }\n\n    /**\n     * Writes the given long to the stream using bits amount of meaningful bits. This command does not\n     * check input values, so if they're larger than what can fit the bits (you should check this before writing),\n     * expect some weird results.\n     *\n     * @param value Value to be written to the stream\n     * @param bits How many bits are stored to the stream\n     */\n    public void writeBits(long value, int bits) {\n        if(bits <= bitsLeft) {\n            int lastBitPosition = bitsLeft - bits;\n            lB |= (value << lastBitPosition) & MASK_ARRAY[bitsLeft - 1];\n            bitsLeft -= bits;\n            checkAndFlipByte(); // We could be at 0 bits left because of the <= condition .. would it be faster with\n                                // the other one?\n        } else {\n            value &= MASK_ARRAY[bits - 1];\n            int firstBitPosition = bits - bitsLeft;\n            lB |= value >>> firstBitPosition;\n            bits -= bitsLeft;\n            flipWord();\n            lB |= value << (64 - bits);\n            bitsLeft -= bits;\n        }\n    }\n\n    /**\n     * Causes the currently handled word to be written to the stream\n     */\n    @Override\n    public void flush() {\n        flipWord();\n    }\n\n    public long[] getLongArray() {\n        long[] copy = Arrays.copyOf(longArray, position + 1);\n        copy[copy.length - 1] = lB;\n        return copy;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/Pair.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\n/**\n * Pair is an extracted timestamp,value pair from the stream\n *\n * @author Michael Burman\n */\npublic class Pair {\n    private long timestamp;\n    private long value;\n\n    public Pair(long timestamp, long value) {\n        this.timestamp = timestamp;\n        this.value = value;\n    }\n\n    public long getTimestamp() {\n        return timestamp;\n    }\n\n    public double getDoubleValue() {\n        return Double.longBitsToDouble(value);\n    }\n\n    public long getLongValue() {\n        return value;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/Predictor.java",
    "content": "/*\n * Copyright 2017 Red Hat, Inc. and/or its affiliates\n * and other contributors as indicated by the @author tags.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage fi.iki.yak.ts.compression.gorilla;\n\n/**\n * @author miburman\n */\npublic interface Predictor {\n\n    /**\n     * Give the real value\n     *\n     * @param value Long / bits of Double\n     */\n    void update(long value);\n\n    /**\n     * Predicts the next value\n     *\n     * @return Predicted value\n     */\n    long predict();\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/ValueCompressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport fi.iki.yak.ts.compression.gorilla.predictors.LastValuePredictor;\n\n/**\n * ValueCompressor for the Gorilla encoding format. Supply with long presentation of the value,\n * in case of doubles use Double.doubleToRawLongBits(value)\n *\n * @author Michael Burman\n */\npublic class ValueCompressor {\n    private int storedLeadingZeros = Integer.MAX_VALUE;\n    private int storedTrailingZeros = 0;\n\n    private Predictor predictor;\n    private BitOutput out;\n\n    public ValueCompressor(BitOutput out) {\n        this(out, new LastValuePredictor());\n    }\n\n    public ValueCompressor(BitOutput out, Predictor predictor) {\n        this.out = out;\n        this.predictor = predictor;\n    }\n\n    void writeFirst(long value) {\n        predictor.update(value);\n        out.writeBits(value, 64);\n    }\n\n    protected void compressValue(long value) {\n        // In original Gorilla, Last-Value predictor is used\n        long diff = predictor.predict() ^ value;\n        predictor.update(value);\n\n        if(diff == 0) {\n            // Write 0\n            out.skipBit();\n        } else {\n            int leadingZeros = Long.numberOfLeadingZeros(diff);\n            int trailingZeros = Long.numberOfTrailingZeros(diff);\n\n            out.writeBit(); // Optimize to writeNewLeading / writeExistingLeading?\n\n            if(leadingZeros >= storedLeadingZeros && trailingZeros >= storedTrailingZeros) {\n                writeExistingLeading(diff);\n            } else {\n                writeNewLeading(diff, leadingZeros, trailingZeros);\n            }\n        }\n    }\n\n    /**\n     * If there at least as many leading zeros and as many trailing zeros as previous value, control bit = 0 (type a)\n     * store the meaningful XORed value\n     *\n     * @param xor XOR between previous value and current\n     */\n    private void writeExistingLeading(long xor) {\n        out.skipBit();\n\n        int significantBits = 64 - storedLeadingZeros - storedTrailingZeros;\n        xor >>>= storedTrailingZeros;\n        out.writeBits(xor, significantBits);\n    }\n\n    /**\n     * store the length of the number of leading zeros in the next 5 bits\n     * store length of the meaningful XORed value in the next 6 bits,\n     * store the meaningful bits of the XORed value\n     * (type b)\n     *\n     * @param xor XOR between previous value and current\n     * @param leadingZeros New leading zeros\n     * @param trailingZeros New trailing zeros\n     */\n    private void writeNewLeading(long xor, int leadingZeros, int trailingZeros) {\n        out.writeBit();\n\n        // Different from version 1.x, use (significantBits - 1) in storage - avoids a branch\n        int significantBits = 64 - leadingZeros - trailingZeros;\n\n        // Different from original, bits 5 -> 6, avoids a branch, allows storing small longs\n        out.writeBits(leadingZeros, 6); // Number of leading zeros in the next 6 bits\n        out.writeBits(significantBits - 1, 6); // Length of meaningful bits in the next 6 bits\n        out.writeBits(xor >>> trailingZeros, significantBits); // Store the meaningful bits of XOR\n\n        storedLeadingZeros = leadingZeros;\n        storedTrailingZeros = trailingZeros;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/ValueDecompressor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport fi.iki.yak.ts.compression.gorilla.predictors.LastValuePredictor;\n\n/**\n * Value decompressor for Gorilla encoded values\n *\n * @author Michael Burman\n */\npublic class ValueDecompressor {\n    private final BitInput in;\n    private final Predictor predictor;\n\n    private int storedLeadingZeros = Integer.MAX_VALUE;\n    private int storedTrailingZeros = 0;\n\n    public ValueDecompressor(BitInput input) {\n        this(input, new LastValuePredictor());\n    }\n\n    public ValueDecompressor(BitInput input, Predictor predictor) {\n        this.in = input;\n        this.predictor = predictor;\n    }\n\n    public long readFirst() {\n        long value = in.getLong(Long.SIZE);\n        predictor.update(value);\n        return value;\n    }\n\n    public long nextValue() {\n        int val = in.nextClearBit(2);\n\n        switch(val) {\n            case 3:\n                // New leading and trailing zeros\n                storedLeadingZeros = (int) in.getLong(6);\n\n                byte significantBits = (byte) in.getLong(6);\n                significantBits++;\n\n                storedTrailingZeros = Long.SIZE - significantBits - storedLeadingZeros;\n                // missing break is intentional, we want to overflow to next one\n            case 2:\n                long value = in.getLong(Long.SIZE - storedLeadingZeros - storedTrailingZeros);\n                value <<= storedTrailingZeros;\n\n                value = predictor.predict() ^ value;\n                predictor.update(value);\n                return value;\n        }\n        return predictor.predict();\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/benchmark/EncodingBenchmark.java",
    "content": "package fi.iki.yak.ts.compression.gorilla.benchmark;\n\nimport fi.iki.yak.ts.compression.gorilla.*;\nimport org.openjdk.jmh.annotations.*;\nimport org.openjdk.jmh.infra.Blackhole;\n\nimport java.nio.ByteBuffer;\nimport java.time.LocalDateTime;\nimport java.time.ZoneOffset;\nimport java.time.temporal.ChronoUnit;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.stream.Stream;\n\n/**\n * @author Michael Burman\n */\n@BenchmarkMode(Mode.Throughput)\n@State(Scope.Benchmark)\n@Fork(1)\n@Warmup(iterations = 5)\n@Measurement(iterations = 10) // Reduce the amount of iterations if you start to see GC interference\npublic class EncodingBenchmark {\n\n    @State(Scope.Benchmark)\n    public static class DataGenerator {\n        public List<Pair> insertList;\n\n        @Param({\"100000\"})\n        public int amountOfPoints;\n\n        public long blockStart;\n\n        public long[] uncompressedTimestamps;\n        public long[] uncompressedValues;\n        public double[] uncompressedDoubles;\n        public long[] compressedArray;\n\n        public ByteBuffer uncompressedBuffer;\n        public ByteBuffer compressedBuffer;\n\n        public List<Pair> pairs;\n\n        @Setup(Level.Trial)\n        public void setup() {\n            blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                    .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n            long now = blockStart + 60;\n            uncompressedTimestamps = new long[amountOfPoints];\n            uncompressedDoubles = new double[amountOfPoints];\n            uncompressedValues = new long[amountOfPoints];\n\n            insertList = new ArrayList<>(amountOfPoints);\n\n            ByteBuffer bb = ByteBuffer.allocate(amountOfPoints * 2*Long.BYTES);\n\n            pairs = new ArrayList<>(amountOfPoints);\n\n            for(int i = 0; i < amountOfPoints; i++) {\n                now += 60;\n                bb.putLong(now);\n                bb.putDouble(i);\n                uncompressedTimestamps[i] = now;\n                uncompressedDoubles[i] = i;\n                uncompressedValues[i] = i;\n                pairs.add(new Pair(now, i));\n//                bb.putLong(i);\n            }\n\n            if (bb.hasArray()) {\n                uncompressedBuffer = bb.duplicate();\n                uncompressedBuffer.flip();\n            }\n            ByteBufferBitOutput output = new ByteBufferBitOutput();\n            LongArrayOutput arrayOutput = new LongArrayOutput(amountOfPoints);\n\n            Compressor c = new Compressor(blockStart, output);\n            GorillaCompressor gc = new GorillaCompressor(blockStart, arrayOutput);\n\n            bb.flip();\n\n            for(int j = 0; j < amountOfPoints; j++) {\n//                c.addValue(bb.getLong(), bb.getLong());\n                c.addValue(bb.getLong(), bb.getDouble());\n                gc.addValue(uncompressedTimestamps[j], uncompressedDoubles[j]);\n            }\n\n            gc.close();\n            c.close();\n\n            ByteBuffer byteBuffer = output.getByteBuffer();\n            byteBuffer.flip();\n            compressedBuffer = byteBuffer;\n\n            compressedArray = arrayOutput.getLongArray();\n        }\n    }\n\n//    @Benchmark\n    @OperationsPerInvocation(100000)\n    public void encodingBenchmark(DataGenerator dg) {\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n        Compressor c = new Compressor(dg.blockStart, output);\n\n        for(int j = 0; j < dg.amountOfPoints; j++) {\n            c.addValue(dg.uncompressedBuffer.getLong(), dg.uncompressedBuffer.getDouble());\n        }\n        c.close();\n        dg.uncompressedBuffer.rewind();\n    }\n\n    @Benchmark\n    @OperationsPerInvocation(100000)\n    public void decodingBenchmark(DataGenerator dg, Blackhole bh) throws Exception {\n        ByteBuffer duplicate = dg.compressedBuffer.duplicate();\n        ByteBufferBitInput input = new ByteBufferBitInput(duplicate);\n        Decompressor d = new Decompressor(input);\n        Pair pair;\n        while((pair = d.readPair()) != null) {\n            bh.consume(pair);\n        }\n    }\n\n    @Benchmark\n    @OperationsPerInvocation(100000)\n    public void encodingGorillaBenchmark(DataGenerator dg) {\n        LongArrayOutput output = new LongArrayOutput();\n        GorillaCompressor c = new GorillaCompressor(dg.blockStart, output);\n\n        for(int j = 0; j < dg.amountOfPoints; j++) {\n            c.addValue(dg.uncompressedTimestamps[j], dg.uncompressedDoubles[j]);\n        }\n        c.close();\n    }\n\n    @Benchmark\n    @OperationsPerInvocation(100000)\n    public void encodingGorillaBenchmarkLong(DataGenerator dg) {\n        LongArrayOutput output = new LongArrayOutput();\n        GorillaCompressor c = new GorillaCompressor(dg.blockStart, output);\n\n        for(int j = 0; j < dg.amountOfPoints; j++) {\n            c.addValue(dg.uncompressedTimestamps[j], dg.uncompressedValues[j]);\n        }\n        c.close();\n    }\n\n//    @Benchmark\n//    @OperationsPerInvocation(100000)\n//    public void encodingGorillaStreamBenchmark(DataGenerator dg) {\n//        LongArrayOutput output = new LongArrayOutput();\n//        GorillaCompressor c = new GorillaCompressor(dg.blockStart, output);\n//\n//        c.compressLongStream(dg.pairs.stream());\n//        c.close();\n//    }\n\n    @Benchmark\n    @OperationsPerInvocation(100000)\n    public void decodingGorillaBenchmark(DataGenerator dg, Blackhole bh) throws Exception {\n        LongArrayInput input = new LongArrayInput(dg.compressedArray);\n        GorillaDecompressor d = new GorillaDecompressor(input);\n        Pair pair;\n        while((pair = d.readPair()) != null) {\n            bh.consume(pair);\n        }\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/DifferentialFCM.java",
    "content": "package fi.iki.yak.ts.compression.gorilla.predictors;\n\nimport fi.iki.yak.ts.compression.gorilla.Predictor;\n\n/**\n * Differential Finite Context Method (DFCM) is a context based predictor.\n *\n * @author Michael Burman\n */\npublic class DifferentialFCM implements Predictor {\n\n    private long lastValue = 0L;\n    private final long[] table;\n    private int lastHash = 0;\n\n    private final int mask;\n\n    /**\n     * Create a new DFCM predictor\n     *\n     * @param size Prediction table size, will be rounded to the next power of two and must be larger than 0\n     */\n    public DifferentialFCM(int size) {\n        if(size > 0) {\n            size--;\n            int leadingZeros = Long.numberOfLeadingZeros(size);\n            int newSize = 1 << (Long.SIZE - leadingZeros);\n\n            this.table = new long[newSize];\n            this.mask = newSize - 1;\n        } else {\n            throw new IllegalArgumentException(\"Size must be positive\");\n        }\n    }\n\n    @Override\n    public void update(long value) {\n        table[lastHash] = value - lastValue;\n        lastHash = (int) (((lastHash << 5) ^ ((value - lastValue) >> 50)) & this.mask);\n        lastValue = value;\n    }\n\n    @Override\n    public long predict() {\n        return table[lastHash] + lastValue;\n    }\n}\n"
  },
  {
    "path": "src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/LastValuePredictor.java",
    "content": "package fi.iki.yak.ts.compression.gorilla.predictors;\n\nimport fi.iki.yak.ts.compression.gorilla.Predictor;\n\n/**\n * Last-Value predictor, a computational predictor using previous value as a prediction for the next one\n *\n * @author Michael Burman\n */\npublic class LastValuePredictor implements Predictor {\n    private long storedVal = 0;\n\n    public LastValuePredictor() {}\n\n    public void update(long value) {\n        this.storedVal = value;\n    }\n\n    public long predict() {\n        return storedVal;\n    }\n}\n"
  },
  {
    "path": "src/test/java/fi/iki/yak/ts/compression/gorilla/EncodeGorillaTest.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertNull;\n\nimport java.nio.ByteBuffer;\nimport java.time.LocalDateTime;\nimport java.time.Month;\nimport java.time.ZoneOffset;\nimport java.time.temporal.ChronoUnit;\nimport java.util.Arrays;\nimport java.util.concurrent.ThreadLocalRandom;\n\nimport org.junit.jupiter.api.Test;\n\nimport fi.iki.yak.ts.compression.gorilla.predictors.DifferentialFCM;\n\n/**\n * These are generic tests to test that input matches the output after compression + decompression cycle, using\n * both the timestamp and value compression.\n *\n * @author Michael Burman\n */\npublic class EncodeGorillaTest {\n\n    private void comparePairsToCompression(long blockTimestamp, Pair[] pairs) {\n        LongArrayOutput output = new LongArrayOutput();\n\n        GorillaCompressor c = new GorillaCompressor(blockTimestamp, output);\n\n        Arrays.stream(pairs).forEach(p -> c.addValue(p.getTimestamp(), p.getDoubleValue()));\n        c.close();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        // Replace with stream once GorillaDecompressor supports it\n        for(int i = 0; i < pairs.length; i++) {\n            Pair pair = d.readPair();\n            assertEquals(pairs[i].getTimestamp(), pair.getTimestamp(), \"Timestamp did not match\");\n            assertEquals(pairs[i].getDoubleValue(), pair.getDoubleValue(), \"Value did not match\");\n        }\n\n        assertNull(d.readPair());\n    }\n\n    @Test\n    void simpleEncodeAndDecodeTest() throws Exception {\n        long now = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        Pair[] pairs = {\n                new Pair(now + 10, Double.doubleToRawLongBits(1.0)),\n                new Pair(now + 20, Double.doubleToRawLongBits(-2.0)),\n                new Pair(now + 28, Double.doubleToRawLongBits(-2.5)),\n                new Pair(now + 84, Double.doubleToRawLongBits(65537)),\n                new Pair(now + 400, Double.doubleToRawLongBits(2147483650.0)),\n                new Pair(now + 2300, Double.doubleToRawLongBits(-16384)),\n                new Pair(now + 16384, Double.doubleToRawLongBits(2.8)),\n                new Pair(now + 16500, Double.doubleToRawLongBits(-38.0))\n        };\n\n        comparePairsToCompression(now, pairs);\n    }\n\n    @Test\n    public void willItBlend() throws Exception {\n        long blockTimestamp = 1500400800000L;\n\n        Pair[] pairs = {\n                new Pair(1500405481623L, 69087),\n                new Pair(1500405488693L, 65640),\n                new Pair(1500405495993L, 58155),\n                new Pair(1500405503743L, 61025),\n                new Pair(1500405511623L, 91156),\n                new Pair(1500405519803L, 37516),\n                new Pair(1500405528313L, 93515),\n                new Pair(1500405537233L, 96226),\n                new Pair(1500405546453L, 23833),\n                new Pair(1500405556103L, 73186),\n                new Pair(1500405566143L, 96947),\n                new Pair(1500405576163L, 46927),\n                new Pair(1500405586173L, 77954),\n                new Pair(1500405596183L, 29302),\n                new Pair(1500405606213L, 6700),\n                new Pair(1500405616163L, 71971),\n                new Pair(1500405625813L, 8528),\n                new Pair(1500405635763L, 85321),\n                new Pair(1500405645634L, 83229),\n                new Pair(1500405655633L, 78298),\n                new Pair(1500405665623L, 87122),\n                new Pair(1500405675623L, 82055),\n                new Pair(1500405685723L, 75067),\n                new Pair(1500405695663L, 33680),\n                new Pair(1500405705743L, 17576),\n                new Pair(1500405715813L, 89701),\n                new Pair(1500405725773L, 21427),\n                new Pair(1500405735883L, 58255),\n                new Pair(1500405745903L, 3768),\n                new Pair(1500405755863L, 62086),\n                new Pair(1500405765843L, 66965),\n                new Pair(1500405775773L, 35801),\n                new Pair(1500405785883L, 72169),\n                new Pair(1500405795843L, 43089),\n                new Pair(1500405805733L, 31418),\n                new Pair(1500405815853L, 84781),\n                new Pair(1500405825963L, 36103),\n                new Pair(1500405836004L, 87431),\n                new Pair(1500405845953L, 7379),\n                new Pair(1500405855913L, 66919),\n                new Pair(1500405865963L, 30906),\n                new Pair(1500405875953L, 88630),\n                new Pair(1500405885943L, 27546),\n                new Pair(1500405896033L, 43813),\n                new Pair(1500405906094L, 2124),\n                new Pair(1500405916063L, 49399),\n                new Pair(1500405926143L, 94577),\n                new Pair(1500405936123L, 98459),\n                new Pair(1500405946033L, 49457),\n                new Pair(1500405956023L, 92838),\n                new Pair(1500405966023L, 15628),\n                new Pair(1500405976043L, 53916),\n                new Pair(1500405986063L, 90387),\n                new Pair(1500405996123L, 43176),\n                new Pair(1500406006123L, 18838),\n                new Pair(1500406016174L, 78847),\n                new Pair(1500406026173L, 39591),\n                new Pair(1500406036004L, 77070),\n                new Pair(1500406045964L, 56788),\n                new Pair(1500406056043L, 96706),\n                new Pair(1500406066123L, 20756),\n                new Pair(1500406076113L, 64433),\n                new Pair(1500406086133L, 45791),\n                new Pair(1500406096123L, 75028),\n                new Pair(1500406106193L, 55403),\n                new Pair(1500406116213L, 36991),\n                new Pair(1500406126073L, 92929),\n                new Pair(1500406136103L, 60416),\n                new Pair(1500406146183L, 55485),\n                new Pair(1500406156383L, 53525),\n                new Pair(1500406166313L, 96021),\n                new Pair(1500406176414L, 22705),\n                new Pair(1500406186613L, 89801),\n                new Pair(1500406196543L, 51975),\n                new Pair(1500406206483L, 86741),\n                new Pair(1500406216483L, 22440),\n                new Pair(1500406226433L, 51818),\n                new Pair(1500406236403L, 61965),\n                new Pair(1500406246413L, 19074),\n                new Pair(1500406256494L, 54521),\n                new Pair(1500406266413L, 59315),\n                new Pair(1500406276303L, 19171),\n                new Pair(1500406286213L, 98800),\n                new Pair(1500406296183L, 7086),\n                new Pair(1500406306103L, 60578),\n                new Pair(1500406316073L, 96828),\n                new Pair(1500406326143L, 83746),\n                new Pair(1500406336153L, 85481),\n                new Pair(1500406346113L, 22346),\n                new Pair(1500406356133L, 80976),\n                new Pair(1500406366065L, 43586),\n                new Pair(1500406376074L, 82500),\n                new Pair(1500406386184L, 13576),\n                new Pair(1500406396113L, 77871),\n                new Pair(1500406406094L, 60978),\n                new Pair(1500406416203L, 35264),\n                new Pair(1500406426323L, 79733),\n                new Pair(1500406436343L, 29140),\n                new Pair(1500406446323L, 7237),\n                new Pair(1500406456344L, 52866),\n                new Pair(1500406466393L, 88456),\n                new Pair(1500406476493L, 33533),\n                new Pair(1500406486524L, 96961),\n                new Pair(1500406496453L, 16389),\n                new Pair(1500406506453L, 31181),\n                new Pair(1500406516433L, 63282),\n                new Pair(1500406526433L, 92857),\n                new Pair(1500406536413L, 4582),\n                new Pair(1500406546383L, 46832),\n                new Pair(1500406556473L, 6335),\n                new Pair(1500406566413L, 44367),\n                new Pair(1500406576513L, 84640),\n                new Pair(1500406586523L, 36174),\n                new Pair(1500406596553L, 40075),\n                new Pair(1500406606603L, 80886),\n                new Pair(1500406616623L, 43784),\n                new Pair(1500406626623L, 25077),\n                new Pair(1500406636723L, 18617),\n                new Pair(1500406646723L, 72681),\n                new Pair(1500406656723L, 84811),\n                new Pair(1500406666783L, 90053),\n                new Pair(1500406676685L, 25708),\n                new Pair(1500406686713L, 57134),\n                new Pair(1500406696673L, 87193),\n                new Pair(1500406706743L, 66057),\n                new Pair(1500406716724L, 51404),\n                new Pair(1500406726753L, 90141),\n                new Pair(1500406736813L, 10434),\n                new Pair(1500406746803L, 29056),\n                new Pair(1500406756833L, 48160),\n                new Pair(1500406766924L, 96652),\n                new Pair(1500406777113L, 64141),\n                new Pair(1500406787113L, 22143),\n                new Pair(1500406797093L, 20561),\n                new Pair(1500406807113L, 66401),\n                new Pair(1500406817283L, 76802),\n                new Pair(1500406827284L, 37555),\n                new Pair(1500406837323L, 63169),\n                new Pair(1500406847463L, 45712),\n                new Pair(1500406857513L, 44751),\n                new Pair(1500406867523L, 98891),\n                new Pair(1500406877523L, 38122),\n                new Pair(1500406887623L, 46202),\n                new Pair(1500406897703L, 5875),\n                new Pair(1500406907663L, 17397),\n                new Pair(1500406917603L, 39994),\n                new Pair(1500406927633L, 82385),\n                new Pair(1500406937623L, 15598),\n                new Pair(1500406947693L, 36235),\n                new Pair(1500406957703L, 97536),\n                new Pair(1500406967673L, 28557),\n                new Pair(1500406977723L, 13985),\n                new Pair(1500406987663L, 64304),\n                new Pair(1500406997573L, 83693),\n                new Pair(1500407007494L, 6574),\n                new Pair(1500407017493L, 25134),\n                new Pair(1500407027503L, 50383),\n                new Pair(1500407037523L, 55922),\n                new Pair(1500407047603L, 73436),\n                new Pair(1500407057473L, 68235),\n                new Pair(1500407067553L, 1469),\n                new Pair(1500407077463L, 44315),\n                new Pair(1500407087463L, 95064),\n                new Pair(1500407097443L, 1997),\n                new Pair(1500407107473L, 17247),\n                new Pair(1500407117453L, 42454),\n                new Pair(1500407127413L, 73631),\n                new Pair(1500407137363L, 96890),\n                new Pair(1500407147343L, 43450),\n                new Pair(1500407157363L, 42042),\n                new Pair(1500407167403L, 83014),\n                new Pair(1500407177473L, 32051),\n                new Pair(1500407187523L, 69280),\n                new Pair(1500407197495L, 21425),\n                new Pair(1500407207453L, 93748),\n                new Pair(1500407217413L, 64151),\n                new Pair(1500407227443L, 38791),\n                new Pair(1500407237463L, 5248),\n                new Pair(1500407247523L, 92935),\n                new Pair(1500407257513L, 18516),\n                new Pair(1500407267584L, 98870),\n                new Pair(1500407277573L, 82244),\n                new Pair(1500407287723L, 65464),\n                new Pair(1500407297723L, 33801),\n                new Pair(1500407307673L, 18331),\n                new Pair(1500407317613L, 89744),\n                new Pair(1500407327553L, 98460),\n                new Pair(1500407337503L, 24709),\n                new Pair(1500407347423L, 8407),\n                new Pair(1500407357383L, 69451),\n                new Pair(1500407367333L, 51100),\n                new Pair(1500407377373L, 25309),\n                new Pair(1500407387443L, 16148),\n                new Pair(1500407397453L, 98974),\n                new Pair(1500407407543L, 80284),\n                new Pair(1500407417583L, 170),\n                new Pair(1500407427453L, 34706),\n                new Pair(1500407437433L, 39681),\n                new Pair(1500407447603L, 6140),\n                new Pair(1500407457513L, 64595),\n                new Pair(1500407467564L, 59862),\n                new Pair(1500407477563L, 53795),\n                new Pair(1500407487593L, 83493),\n                new Pair(1500407497584L, 90639),\n                new Pair(1500407507623L, 16777),\n                new Pair(1500407517613L, 11096),\n                new Pair(1500407527673L, 38512),\n                new Pair(1500407537963L, 52759),\n                new Pair(1500407548023L, 79567),\n                new Pair(1500407558033L, 48664),\n                new Pair(1500407568113L, 10710),\n                new Pair(1500407578164L, 25635),\n                new Pair(1500407588213L, 40985),\n                new Pair(1500407598163L, 94089),\n                new Pair(1500407608163L, 50056),\n                new Pair(1500407618223L, 15550),\n                new Pair(1500407628143L, 78823),\n                new Pair(1500407638223L, 9044),\n                new Pair(1500407648173L, 20782),\n                new Pair(1500407658023L, 86390),\n                new Pair(1500407667903L, 79444),\n                new Pair(1500407677903L, 84051),\n                new Pair(1500407687923L, 91554),\n                new Pair(1500407697913L, 58777),\n                new Pair(1500407708003L, 89474),\n                new Pair(1500407718083L, 94026),\n                new Pair(1500407728034L, 41613),\n                new Pair(1500407738083L, 64667),\n                new Pair(1500407748034L, 5160),\n                new Pair(1500407758003L, 45140),\n                new Pair(1500407768033L, 53704),\n                new Pair(1500407778083L, 68097),\n                new Pair(1500407788043L, 81137),\n                new Pair(1500407798023L, 59657),\n                new Pair(1500407808033L, 56572),\n                new Pair(1500407817983L, 1993),\n                new Pair(1500407828063L, 62608),\n                new Pair(1500407838213L, 76489),\n                new Pair(1500407848203L, 22147),\n                new Pair(1500407858253L, 92829),\n                new Pair(1500407868073L, 48499),\n                new Pair(1500407878053L, 89152),\n                new Pair(1500407888073L, 9191),\n                new Pair(1500407898033L, 49881),\n                new Pair(1500407908113L, 96020),\n                new Pair(1500407918213L, 90203),\n                new Pair(1500407928234L, 32217),\n                new Pair(1500407938253L, 94302),\n                new Pair(1500407948293L, 83111),\n                new Pair(1500407958234L, 75576),\n                new Pair(1500407968073L, 5973),\n                new Pair(1500407978023L, 5175),\n                new Pair(1500407987923L, 63350),\n                new Pair(1500407997833L, 44081)\n        };\n\n        comparePairsToCompression(blockTimestamp, pairs);\n    }\n\n    /**\n     * Tests encoding of similar floats, see https://github.com/dgryski/go-tsz/issues/4 for more information.\n     */\n    @Test\n    void testEncodeSimilarFloats() throws Exception {\n        long now = LocalDateTime.of(2015, Month.MARCH, 02, 00, 00).toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        LongArrayOutput output = new LongArrayOutput();\n        GorillaCompressor c = new GorillaCompressor(now, output);\n\n        ByteBuffer bb = ByteBuffer.allocate(5 * 2*Long.BYTES);\n\n        bb.putLong(now + 1);\n        bb.putDouble(6.00065e+06);\n        bb.putLong(now + 2);\n        bb.putDouble(6.000656e+06);\n        bb.putLong(now + 3);\n        bb.putDouble(6.000657e+06);\n        bb.putLong(now + 4);\n        bb.putDouble(6.000659e+06);\n        bb.putLong(now + 5);\n        bb.putDouble(6.000661e+06);\n\n        bb.flip();\n\n        for(int j = 0; j < 5; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        // Replace with stream once GorillaDecompressor supports it\n        for(int i = 0; i < 5; i++) {\n            Pair pair = d.readPair();\n            assertEquals(bb.getLong(), pair.getTimestamp(), \"Timestamp did not match\");\n            assertEquals(bb.getDouble(), pair.getDoubleValue(), \"Value did not match\");\n        }\n        assertNull(d.readPair());\n    }\n\n    /**\n     * Tests writing enough large amount of datapoints that causes the included LongArrayOutput to do\n     * internal byte array expansion.\n     */\n    @Test\n    void testEncodeLargeAmountOfData() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 100000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        LongArrayOutput output = new LongArrayOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putDouble(i * Math.random());\n        }\n\n        GorillaCompressor c = new GorillaCompressor(blockStart, output);\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            double val = bb.getDouble();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getDoubleValue());\n        }\n        assertNull(d.readPair());\n    }\n\n    @Test\n    void testEncodeLargeAmountOfDataOldBuffer() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 100000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putDouble(i * Math.random());\n        }\n\n        GorillaCompressor c = new GorillaCompressor(blockStart, output);\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            double val = bb.getDouble();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getDoubleValue());\n        }\n        assertNull(d.readPair());\n    }\n\n    /**\n     * Although not intended usage, an empty block should not cause errors\n     */\n    @Test\n    void testEmptyBlock() throws Exception {\n        long now = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        LongArrayOutput output = new LongArrayOutput();\n\n        GorillaCompressor c = new GorillaCompressor(now, output);\n        c.close();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        assertNull(d.readPair());\n    }\n\n    @Test\n    void testCopyFlush() {\n        long now = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        LongArrayOutput output = new LongArrayOutput();\n\n        GorillaCompressor c = new GorillaCompressor(now, output);\n\n        c.addValue(now + 1, 1.0);\n        c.addValue(now + 2, 1.0);\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        assertEquals(now + 1, d.readPair().getTimestamp());\n        assertEquals(now + 2, d.readPair().getTimestamp());\n    }\n\n    /**\n     * Long values should be compressable and decompressable in the stream\n     */\n    @Test\n    void testLongEncoding() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 10000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        LongArrayOutput output = new LongArrayOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putLong(ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE));\n        }\n\n        GorillaCompressor c = new GorillaCompressor(blockStart, output);\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getLong());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            long val = bb.getLong();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getLongValue());\n        }\n        assertNull(d.readPair());\n    }\n\n    /**\n     * Tests writing enough large amount of datapoints that causes the included LongArrayOutput to do\n     * internal byte array expansion.\n     */\n    @Test\n    void testDifferentialFCM() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 100000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        LongArrayOutput output = new LongArrayOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putDouble(i * Math.random());\n        }\n\n        GorillaCompressor c = new GorillaCompressor(blockStart, output, new DifferentialFCM(1024));\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        LongArrayInput input = new LongArrayInput(output.getLongArray());\n        GorillaDecompressor d = new GorillaDecompressor(input, new DifferentialFCM(1024));\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            double val = bb.getDouble();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getDoubleValue());\n        }\n        assertNull(d.readPair());\n    }\n\n}\n"
  },
  {
    "path": "src/test/java/fi/iki/yak/ts/compression/gorilla/EncodeTest.java",
    "content": "package fi.iki.yak.ts.compression.gorilla;\n\nimport static org.junit.jupiter.api.Assertions.assertEquals;\nimport static org.junit.jupiter.api.Assertions.assertNull;\n\nimport java.nio.ByteBuffer;\nimport java.time.LocalDateTime;\nimport java.time.Month;\nimport java.time.ZoneOffset;\nimport java.time.temporal.ChronoUnit;\nimport java.util.Arrays;\nimport java.util.concurrent.ThreadLocalRandom;\n\nimport org.junit.jupiter.api.Test;\n\n/**\n * These are generic tests to test that input matches the output after compression + decompression cycle, using\n * both the timestamp and value compression.\n *\n * @author Michael Burman\n */\npublic class EncodeTest {\n\n    private void comparePairsToCompression(long blockTimestamp, Pair[] pairs) {\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n        Compressor c = new Compressor(blockTimestamp, output);\n        Arrays.stream(pairs).forEach(p -> c.addValue(p.getTimestamp(), p.getDoubleValue()));\n        c.close();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        Decompressor d = new Decompressor(input);\n\n        // Replace with stream once decompressor supports it\n        for(int i = 0; i < pairs.length; i++) {\n            Pair pair = d.readPair();\n            assertEquals(pairs[i].getTimestamp(), pair.getTimestamp(), \"Timestamp did not match\");\n            assertEquals(pairs[i].getDoubleValue(), pair.getDoubleValue(), \"Value did not match\");\n        }\n\n        assertNull(d.readPair());\n    }\n\n    @Test\n    void simpleEncodeAndDecodeTest() throws Exception {\n        long now = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        Pair[] pairs = {\n                new Pair(now + 10, Double.doubleToRawLongBits(1.0)),\n                new Pair(now + 20, Double.doubleToRawLongBits(-2.0)),\n                new Pair(now + 28, Double.doubleToRawLongBits(-2.5)),\n                new Pair(now + 84, Double.doubleToRawLongBits(65537)),\n                new Pair(now + 400, Double.doubleToRawLongBits(2147483650.0)),\n                new Pair(now + 2300, Double.doubleToRawLongBits(-16384)),\n                new Pair(now + 16384, Double.doubleToRawLongBits(2.8)),\n                new Pair(now + 16500, Double.doubleToRawLongBits(-38.0))\n        };\n\n        comparePairsToCompression(now, pairs);\n    }\n\n    @Test\n    public void willItBlend() throws Exception {\n        long blockTimestamp = 1500400800000L;\n\n        Pair[] pairs = {\n                new Pair(1500405481623L, 69087),\n                new Pair(1500405488693L, 65640),\n                new Pair(1500405495993L, 58155),\n                new Pair(1500405503743L, 61025),\n                new Pair(1500405511623L, 91156),\n                new Pair(1500405519803L, 37516),\n                new Pair(1500405528313L, 93515),\n                new Pair(1500405537233L, 96226),\n                new Pair(1500405546453L, 23833),\n                new Pair(1500405556103L, 73186),\n                new Pair(1500405566143L, 96947),\n                new Pair(1500405576163L, 46927),\n                new Pair(1500405586173L, 77954),\n                new Pair(1500405596183L, 29302),\n                new Pair(1500405606213L, 6700),\n                new Pair(1500405616163L, 71971),\n                new Pair(1500405625813L, 8528),\n                new Pair(1500405635763L, 85321),\n                new Pair(1500405645634L, 83229),\n                new Pair(1500405655633L, 78298),\n                new Pair(1500405665623L, 87122),\n                new Pair(1500405675623L, 82055),\n                new Pair(1500405685723L, 75067),\n                new Pair(1500405695663L, 33680),\n                new Pair(1500405705743L, 17576),\n                new Pair(1500405715813L, 89701),\n                new Pair(1500405725773L, 21427),\n                new Pair(1500405735883L, 58255),\n                new Pair(1500405745903L, 3768),\n                new Pair(1500405755863L, 62086),\n                new Pair(1500405765843L, 66965),\n                new Pair(1500405775773L, 35801),\n                new Pair(1500405785883L, 72169),\n                new Pair(1500405795843L, 43089),\n                new Pair(1500405805733L, 31418),\n                new Pair(1500405815853L, 84781),\n                new Pair(1500405825963L, 36103),\n                new Pair(1500405836004L, 87431),\n                new Pair(1500405845953L, 7379),\n                new Pair(1500405855913L, 66919),\n                new Pair(1500405865963L, 30906),\n                new Pair(1500405875953L, 88630),\n                new Pair(1500405885943L, 27546),\n                new Pair(1500405896033L, 43813),\n                new Pair(1500405906094L, 2124),\n                new Pair(1500405916063L, 49399),\n                new Pair(1500405926143L, 94577),\n                new Pair(1500405936123L, 98459),\n                new Pair(1500405946033L, 49457),\n                new Pair(1500405956023L, 92838),\n                new Pair(1500405966023L, 15628),\n                new Pair(1500405976043L, 53916),\n                new Pair(1500405986063L, 90387),\n                new Pair(1500405996123L, 43176),\n                new Pair(1500406006123L, 18838),\n                new Pair(1500406016174L, 78847),\n                new Pair(1500406026173L, 39591),\n                new Pair(1500406036004L, 77070),\n                new Pair(1500406045964L, 56788),\n                new Pair(1500406056043L, 96706),\n                new Pair(1500406066123L, 20756),\n                new Pair(1500406076113L, 64433),\n                new Pair(1500406086133L, 45791),\n                new Pair(1500406096123L, 75028),\n                new Pair(1500406106193L, 55403),\n                new Pair(1500406116213L, 36991),\n                new Pair(1500406126073L, 92929),\n                new Pair(1500406136103L, 60416),\n                new Pair(1500406146183L, 55485),\n                new Pair(1500406156383L, 53525),\n                new Pair(1500406166313L, 96021),\n                new Pair(1500406176414L, 22705),\n                new Pair(1500406186613L, 89801),\n                new Pair(1500406196543L, 51975),\n                new Pair(1500406206483L, 86741),\n                new Pair(1500406216483L, 22440),\n                new Pair(1500406226433L, 51818),\n                new Pair(1500406236403L, 61965),\n                new Pair(1500406246413L, 19074),\n                new Pair(1500406256494L, 54521),\n                new Pair(1500406266413L, 59315),\n                new Pair(1500406276303L, 19171),\n                new Pair(1500406286213L, 98800),\n                new Pair(1500406296183L, 7086),\n                new Pair(1500406306103L, 60578),\n                new Pair(1500406316073L, 96828),\n                new Pair(1500406326143L, 83746),\n                new Pair(1500406336153L, 85481),\n                new Pair(1500406346113L, 22346),\n                new Pair(1500406356133L, 80976),\n                new Pair(1500406366065L, 43586),\n                new Pair(1500406376074L, 82500),\n                new Pair(1500406386184L, 13576),\n                new Pair(1500406396113L, 77871),\n                new Pair(1500406406094L, 60978),\n                new Pair(1500406416203L, 35264),\n                new Pair(1500406426323L, 79733),\n                new Pair(1500406436343L, 29140),\n                new Pair(1500406446323L, 7237),\n                new Pair(1500406456344L, 52866),\n                new Pair(1500406466393L, 88456),\n                new Pair(1500406476493L, 33533),\n                new Pair(1500406486524L, 96961),\n                new Pair(1500406496453L, 16389),\n                new Pair(1500406506453L, 31181),\n                new Pair(1500406516433L, 63282),\n                new Pair(1500406526433L, 92857),\n                new Pair(1500406536413L, 4582),\n                new Pair(1500406546383L, 46832),\n                new Pair(1500406556473L, 6335),\n                new Pair(1500406566413L, 44367),\n                new Pair(1500406576513L, 84640),\n                new Pair(1500406586523L, 36174),\n                new Pair(1500406596553L, 40075),\n                new Pair(1500406606603L, 80886),\n                new Pair(1500406616623L, 43784),\n                new Pair(1500406626623L, 25077),\n                new Pair(1500406636723L, 18617),\n                new Pair(1500406646723L, 72681),\n                new Pair(1500406656723L, 84811),\n                new Pair(1500406666783L, 90053),\n                new Pair(1500406676685L, 25708),\n                new Pair(1500406686713L, 57134),\n                new Pair(1500406696673L, 87193),\n                new Pair(1500406706743L, 66057),\n                new Pair(1500406716724L, 51404),\n                new Pair(1500406726753L, 90141),\n                new Pair(1500406736813L, 10434),\n                new Pair(1500406746803L, 29056),\n                new Pair(1500406756833L, 48160),\n                new Pair(1500406766924L, 96652),\n                new Pair(1500406777113L, 64141),\n                new Pair(1500406787113L, 22143),\n                new Pair(1500406797093L, 20561),\n                new Pair(1500406807113L, 66401),\n                new Pair(1500406817283L, 76802),\n                new Pair(1500406827284L, 37555),\n                new Pair(1500406837323L, 63169),\n                new Pair(1500406847463L, 45712),\n                new Pair(1500406857513L, 44751),\n                new Pair(1500406867523L, 98891),\n                new Pair(1500406877523L, 38122),\n                new Pair(1500406887623L, 46202),\n                new Pair(1500406897703L, 5875),\n                new Pair(1500406907663L, 17397),\n                new Pair(1500406917603L, 39994),\n                new Pair(1500406927633L, 82385),\n                new Pair(1500406937623L, 15598),\n                new Pair(1500406947693L, 36235),\n                new Pair(1500406957703L, 97536),\n                new Pair(1500406967673L, 28557),\n                new Pair(1500406977723L, 13985),\n                new Pair(1500406987663L, 64304),\n                new Pair(1500406997573L, 83693),\n                new Pair(1500407007494L, 6574),\n                new Pair(1500407017493L, 25134),\n                new Pair(1500407027503L, 50383),\n                new Pair(1500407037523L, 55922),\n                new Pair(1500407047603L, 73436),\n                new Pair(1500407057473L, 68235),\n                new Pair(1500407067553L, 1469),\n                new Pair(1500407077463L, 44315),\n                new Pair(1500407087463L, 95064),\n                new Pair(1500407097443L, 1997),\n                new Pair(1500407107473L, 17247),\n                new Pair(1500407117453L, 42454),\n                new Pair(1500407127413L, 73631),\n                new Pair(1500407137363L, 96890),\n                new Pair(1500407147343L, 43450),\n                new Pair(1500407157363L, 42042),\n                new Pair(1500407167403L, 83014),\n                new Pair(1500407177473L, 32051),\n                new Pair(1500407187523L, 69280),\n                new Pair(1500407197495L, 21425),\n                new Pair(1500407207453L, 93748),\n                new Pair(1500407217413L, 64151),\n                new Pair(1500407227443L, 38791),\n                new Pair(1500407237463L, 5248),\n                new Pair(1500407247523L, 92935),\n                new Pair(1500407257513L, 18516),\n                new Pair(1500407267584L, 98870),\n                new Pair(1500407277573L, 82244),\n                new Pair(1500407287723L, 65464),\n                new Pair(1500407297723L, 33801),\n                new Pair(1500407307673L, 18331),\n                new Pair(1500407317613L, 89744),\n                new Pair(1500407327553L, 98460),\n                new Pair(1500407337503L, 24709),\n                new Pair(1500407347423L, 8407),\n                new Pair(1500407357383L, 69451),\n                new Pair(1500407367333L, 51100),\n                new Pair(1500407377373L, 25309),\n                new Pair(1500407387443L, 16148),\n                new Pair(1500407397453L, 98974),\n                new Pair(1500407407543L, 80284),\n                new Pair(1500407417583L, 170),\n                new Pair(1500407427453L, 34706),\n                new Pair(1500407437433L, 39681),\n                new Pair(1500407447603L, 6140),\n                new Pair(1500407457513L, 64595),\n                new Pair(1500407467564L, 59862),\n                new Pair(1500407477563L, 53795),\n                new Pair(1500407487593L, 83493),\n                new Pair(1500407497584L, 90639),\n                new Pair(1500407507623L, 16777),\n                new Pair(1500407517613L, 11096),\n                new Pair(1500407527673L, 38512),\n                new Pair(1500407537963L, 52759),\n                new Pair(1500407548023L, 79567),\n                new Pair(1500407558033L, 48664),\n                new Pair(1500407568113L, 10710),\n                new Pair(1500407578164L, 25635),\n                new Pair(1500407588213L, 40985),\n                new Pair(1500407598163L, 94089),\n                new Pair(1500407608163L, 50056),\n                new Pair(1500407618223L, 15550),\n                new Pair(1500407628143L, 78823),\n                new Pair(1500407638223L, 9044),\n                new Pair(1500407648173L, 20782),\n                new Pair(1500407658023L, 86390),\n                new Pair(1500407667903L, 79444),\n                new Pair(1500407677903L, 84051),\n                new Pair(1500407687923L, 91554),\n                new Pair(1500407697913L, 58777),\n                new Pair(1500407708003L, 89474),\n                new Pair(1500407718083L, 94026),\n                new Pair(1500407728034L, 41613),\n                new Pair(1500407738083L, 64667),\n                new Pair(1500407748034L, 5160),\n                new Pair(1500407758003L, 45140),\n                new Pair(1500407768033L, 53704),\n                new Pair(1500407778083L, 68097),\n                new Pair(1500407788043L, 81137),\n                new Pair(1500407798023L, 59657),\n                new Pair(1500407808033L, 56572),\n                new Pair(1500407817983L, 1993),\n                new Pair(1500407828063L, 62608),\n                new Pair(1500407838213L, 76489),\n                new Pair(1500407848203L, 22147),\n                new Pair(1500407858253L, 92829),\n                new Pair(1500407868073L, 48499),\n                new Pair(1500407878053L, 89152),\n                new Pair(1500407888073L, 9191),\n                new Pair(1500407898033L, 49881),\n                new Pair(1500407908113L, 96020),\n                new Pair(1500407918213L, 90203),\n                new Pair(1500407928234L, 32217),\n                new Pair(1500407938253L, 94302),\n                new Pair(1500407948293L, 83111),\n                new Pair(1500407958234L, 75576),\n                new Pair(1500407968073L, 5973),\n                new Pair(1500407978023L, 5175),\n                new Pair(1500407987923L, 63350),\n                new Pair(1500407997833L, 44081)\n        };\n\n        comparePairsToCompression(blockTimestamp, pairs);\n    }\n\n    /**\n     * Tests encoding of similar floats, see https://github.com/dgryski/go-tsz/issues/4 for more information.\n     */\n    @Test\n    void testEncodeSimilarFloats() throws Exception {\n        long now = LocalDateTime.of(2015, Month.MARCH, 02, 00, 00).toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n        Compressor c = new Compressor(now, output);\n\n        ByteBuffer bb = ByteBuffer.allocate(5 * 2*Long.BYTES);\n\n        bb.putLong(now + 1);\n        bb.putDouble(6.00065e+06);\n        bb.putLong(now + 2);\n        bb.putDouble(6.000656e+06);\n        bb.putLong(now + 3);\n        bb.putDouble(6.000657e+06);\n        bb.putLong(now + 4);\n        bb.putDouble(6.000659e+06);\n        bb.putLong(now + 5);\n        bb.putDouble(6.000661e+06);\n\n        bb.flip();\n\n        for(int j = 0; j < 5; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        Decompressor d = new Decompressor(input);\n\n        // Replace with stream once decompressor supports it\n        for(int i = 0; i < 5; i++) {\n            Pair pair = d.readPair();\n            assertEquals(bb.getLong(), pair.getTimestamp(), \"Timestamp did not match\");\n            assertEquals(bb.getDouble(), pair.getDoubleValue(), \"Value did not match\");\n        }\n        assertNull(d.readPair());\n    }\n\n    /**\n     * Tests writing enough large amount of datapoints that causes the included ByteBufferBitOutput to do\n     * internal byte array expansion.\n     */\n    @Test\n    void testEncodeLargeAmountOfData() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 100000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putDouble(i * Math.random());\n        }\n\n        Compressor c = new Compressor(blockStart, output);\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getDouble());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        Decompressor d = new Decompressor(input);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            double val = bb.getDouble();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getDoubleValue());\n        }\n        assertNull(d.readPair());\n    }\n\n    /**\n     * Although not intended usage, an empty block should not cause errors\n     */\n    @Test\n    void testEmptyBlock() throws Exception {\n        long now = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n\n        Compressor c = new Compressor(now, output);\n        c.close();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        Decompressor d = new Decompressor(input);\n\n        assertNull(d.readPair());\n    }\n\n    @Test\n    void testLongEncoding() throws Exception {\n        // This test should trigger ByteBuffer reallocation\n        int amountOfPoints = 10000;\n        long blockStart = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS)\n                .toInstant(ZoneOffset.UTC).toEpochMilli();\n        ByteBufferBitOutput output = new ByteBufferBitOutput();\n\n        long now = blockStart + 60;\n        ByteBuffer bb = ByteBuffer.allocateDirect(amountOfPoints * 2*Long.BYTES);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            bb.putLong(now + i*60);\n            bb.putLong(ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE));\n        }\n\n        Compressor c = new Compressor(blockStart, output);\n\n        bb.flip();\n\n        for(int j = 0; j < amountOfPoints; j++) {\n            c.addValue(bb.getLong(), bb.getLong());\n        }\n\n        c.close();\n\n        bb.flip();\n\n        ByteBuffer byteBuffer = output.getByteBuffer();\n        byteBuffer.flip();\n\n        ByteBufferBitInput input = new ByteBufferBitInput(byteBuffer);\n        Decompressor d = new Decompressor(input);\n\n        for(int i = 0; i < amountOfPoints; i++) {\n            long tStamp = bb.getLong();\n            long val = bb.getLong();\n            Pair pair = d.readPair();\n            assertEquals(tStamp, pair.getTimestamp(), \"Expected timestamp did not match at point \" + i);\n            assertEquals(val, pair.getLongValue());\n        }\n        assertNull(d.readPair());\n    }\n}\n"
  }
]