Repository: apache/arrow-julia Branch: main Commit: 5ade01aca6de Files: 80 Total size: 1.3 MB Directory structure: gitextract_uitozoi5/ ├── .JuliaFormatter.toml ├── .asf.yaml ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── CompatHelper.yml │ ├── TagBot.yml │ ├── ci.yml │ └── ci_nightly.yml ├── .gitignore ├── LICENSE ├── NOTICE ├── Project.toml ├── README.md ├── codecov.yaml ├── dev/ │ └── release/ │ ├── .dir-locals.el │ ├── .gitignore │ ├── README.md │ ├── check_rat_report.py │ ├── rat_exclude_files.txt │ ├── release.sh │ ├── release_rc.sh │ ├── run_rat.sh │ └── verify_rc.sh ├── docs/ │ ├── .gitignore │ ├── Project.toml │ ├── make.jl │ └── src/ │ ├── index.md │ ├── manual.md │ └── reference.md ├── src/ │ ├── Arrow.jl │ ├── ArrowTypes/ │ │ ├── LICENSE.md │ │ ├── Project.toml │ │ ├── src/ │ │ │ └── ArrowTypes.jl │ │ └── test/ │ │ ├── Project.toml │ │ ├── runtests.jl │ │ └── tests.jl │ ├── FlatBuffers/ │ │ ├── FlatBuffers.jl │ │ ├── builder.jl │ │ └── table.jl │ ├── append.jl │ ├── arraytypes/ │ │ ├── arraytypes.jl │ │ ├── bool.jl │ │ ├── compressed.jl │ │ ├── dictencoding.jl │ │ ├── fixedsizelist.jl │ │ ├── list.jl │ │ ├── map.jl │ │ ├── primitive.jl │ │ ├── struct.jl │ │ ├── unions.jl │ │ └── views.jl │ ├── eltypes.jl │ ├── metadata/ │ │ ├── File.jl │ │ ├── Flatbuf.jl │ │ ├── Message.jl │ │ └── Schema.jl │ ├── show.jl │ ├── table.jl │ ├── utils.jl │ └── write.jl └── test/ ├── Project.toml ├── arrowjson/ │ ├── datetime.json │ ├── decimal.json │ ├── dictionary.json │ ├── dictionary_unsigned.json │ ├── map.json │ ├── nested.json │ ├── primitive-empty.json │ ├── primitive.json │ └── primitive_no_batches.json ├── arrowjson.jl ├── dates.jl ├── integrationtest.jl ├── java_compress_len_neg_one.arrow ├── java_compressed_zero_length.arrow ├── old_zdt.arrow ├── pyarrow_roundtrip.jl ├── reject_reason_trimmed.arrow ├── runtests.jl ├── testappend.jl └── testtables.jl ================================================ FILE CONTENTS ================================================ ================================================ FILE: .JuliaFormatter.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # https://github.com/domluna/JuliaFormatter.jl/blob/master/README.md whitespace_ops_in_indices = true remove_extra_newlines = true whitespace_in_kwargs = false ================================================ FILE: .asf.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features github: description: "Official Julia implementation of Apache Arrow" homepage: https://arrow.apache.org/julia/ labels: - apache-arrow - julia features: issues: true discussions: true enabled_merge_buttons: merge: false rebase: true squash: true protected_branches: main: required_linear_history: true notifications: commits: commits@arrow.apache.org issues_status: issues@arrow.apache.org issues_comment: github@arrow.apache.org pullrequests: github@arrow.apache.org discussions: user@arrow.apache.org # publishes the content of the `asf-site` branch to # https://arrow.apache.org/julia/ publish: whoami: asf-site subdir: julia ================================================ FILE: .github/dependabot.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/CompatHelper.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: CompatHelper on: schedule: - cron: 0 0 * * * workflow_dispatch: permissions: contents: write pull-requests: write jobs: CompatHelper: runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Check if Julia is already available in the PATH id: julia_in_path run: which julia continue-on-error: true - name: Install Julia, but only if it is not already available in the PATH uses: julia-actions/setup-julia@v3 with: version: '1' arch: ${{ runner.arch }} if: steps.julia_in_path.outcome != 'success' - name: "Add the General registry via Git" run: | import Pkg ENV["JULIA_PKG_SERVER"] = "" Pkg.Registry.add("General") shell: julia --color=yes {0} - name: "Install CompatHelper" run: | import Pkg name = "CompatHelper" uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" version = "3" Pkg.add(; name, uuid, version) shell: julia --color=yes {0} - name: "Run CompatHelper" run: | import CompatHelper CompatHelper.main() shell: julia --color=yes {0} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} ================================================ FILE: .github/workflows/TagBot.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: TagBot on: issue_comment: types: - created workflow_dispatch: jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: JuliaRegistries/TagBot@v1 with: token: ${{ secrets.GITHUB_TOKEN }} ssh: ${{ secrets.DOCUMENTER_KEY }} ================================================ FILE: .github/workflows/ci.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: CI on: pull_request: push: branches: - '**' - '!dependabot/**' tags: - '**' jobs: license: name: Audit licenses runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: '3.x' - name: Run Release audit tool (Rat) run: dev/release/run_rat.sh . release: name: Verify release - ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: os: - macos-latest - ubuntu-latest steps: - uses: actions/checkout@v6 - name: Install dependencies if: matrix.os == 'macos-latest' run: | brew install julia subversion - name: Install dependencies if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install -y -V subversion - name: Create run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" RELEASE_DEFAULT=0 dev/release/release_rc.sh 100 - uses: actions/cache@v5 with: path: ~/.julia/artifacts key: ${{ runner.os }}-release-${{ hashFiles('**/Project.toml') }} restore-keys: | ${{ runner.os }}-release- - name: Verify run: | version=$(grep -o '^version = ".*"' "Project.toml" | \ sed -e 's/^version = "//g' \ -e 's/"$//g') VERIFY_DEFAULT=0 dev/release/verify_rc.sh ${version} 100 test: name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.nthreads }} threads - ${{ github.event_name }} runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: fail-fast: false matrix: pkg: - name: Arrow.jl dir: '.' - name: ArrowTypes.jl dir: './src/ArrowTypes' version: - 'min' - 'lts' - '1' # automatically expands to the latest stable 1.x release of Julia - 'pre' # expands to latest alpha, beta or RC, if available, otherwise same as `1` os: - ubuntu-latest - macos-latest nthreads: - 1 - 2 steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} - uses: actions/cache@v5 env: cache-name: cache-artifacts with: path: ~/.julia/artifacts key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} restore-keys: | ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1.6 with: project: ${{ matrix.pkg.dir }} - name: Dev local ArrowTypes for Arrow.jl tests if: matrix.pkg.name == 'Arrow.jl' shell: julia --project=. {0} run: | using Pkg Pkg.develop(PackageSpec(path="src/ArrowTypes")) - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} with: project: ${{ matrix.pkg.dir }} - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v6 with: files: lcov.info test_monorepo: name: Monorepo dev - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: true matrix: version: - '1' # automatically expands to the latest stable 1.x release of Julia os: - ubuntu-latest arch: - x64 steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: actions/cache@v5 env: cache-name: cache-artifacts with: path: ~/.julia/artifacts key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} restore-keys: | ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - name: Dev monorepo dependencies shell: julia --project=monorepo {0} run: | using Pkg; Pkg.develop([PackageSpec(path="."), PackageSpec(path="src/ArrowTypes")]) - name: Run monorepo tests continue-on-error: false run: > julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("Arrow")' docs: name: Documentation runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v6 - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-docdeploy@latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} Format: runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: julia-actions/setup-julia@v3 - uses: actions/checkout@v6 - name: Install JuliaFormatter and format # This will use the latest version by default but you can set the version like so: # julia -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.13.0"))' run: | julia -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))' julia -e 'using JuliaFormatter; format(".", verbose=true)' - name: Format check run: | julia -e ' out = Cmd(`git diff --name-only`) |> read |> String if out == "" exit(0) else @error "Some files have not been formatted !!!" write(stdout, out) exit(1) end' ================================================ FILE: .github/workflows/ci_nightly.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Nightly on: schedule: - cron: 0 0 * * 1 # run once a week on Mondays workflow_dispatch: jobs: test: name: ${{ matrix.pkg.name }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.nthreads }} threads - ${{ github.event_name }} runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: fail-fast: false matrix: pkg: - name: Arrow.jl dir: '.' - name: ArrowTypes.jl dir: './src/ArrowTypes' version: - 'nightly' os: - ubuntu-latest arch: - x64 nthreads: [1, 2] steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: actions/cache@v5 env: cache-name: cache-artifacts with: path: ~/.julia/artifacts key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} restore-keys: | ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1.6 with: project: ${{ matrix.pkg.dir }} - uses: julia-actions/julia-runtest@v1 env: JULIA_NUM_THREADS: ${{ matrix.nthreads }} with: project: ${{ matrix.pkg.dir }} - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v6 with: files: lcov.info test_monorepo: name: Monorepo dev - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: true matrix: version: - 'nightly' os: - ubuntu-latest arch: - x64 steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: actions/cache@v5 env: cache-name: cache-artifacts with: path: ~/.julia/artifacts key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} restore-keys: | ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - name: Dev monorepo dependencies shell: julia --project=monorepo {0} run: | using Pkg; Pkg.develop([PackageSpec(path="."), PackageSpec(path="src/ArrowTypes")]) - name: Run monorepo tests continue-on-error: false run: > julia --color=yes --project=monorepo -e 'using Pkg; Pkg.test("Arrow")' ================================================ FILE: .gitignore ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. Manifest.toml Manifest-v*.toml .DS_STORE *.jl.cov *.jl.*.cov *.jl.mem test/_scrap.jl .DS_STORE ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE ================================================ Apache Arrow Julia Copyright 2016-2025 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ================================================ FILE: Project.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. name = "Arrow" uuid = "69666777-d1a9-59fb-9406-91d4454c9d45" authors = ["quinnj "] version = "2.8.1" [deps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1" CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2" ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" EnumX = "4e289a0a-7415-4d19-859d-a7e5c4648b56" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" StringViews = "354b36f9-a18e-4713-926e-db85100087ba" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] ArrowTypes = "1.1,2" BitIntegers = "0.2, 0.3" CodecLz4 = "0.4" CodecZstd = "0.7, 0.8" ConcurrentUtilities = "2" DataAPI = "1" EnumX = "1" PooledArrays = "0.5, 1.0" SentinelArrays = "1" StringViews = "1" Tables = "1.1" TimeZones = "1" TranscodingStreams = "0.9.12, 0.10, 0.11" julia = "1.9" ================================================ FILE: README.md ================================================ # Arrow [![docs](https://img.shields.io/badge/docs-latest-blue&logo=julia)](https://arrow.apache.org/julia/) [![CI](https://github.com/apache/arrow-julia/workflows/CI/badge.svg)](https://github.com/apache/arrow-julia/actions?query=workflow%3ACI) [![codecov](https://app.codecov.io/gh/apache/arrow-julia/branch/main/graph/badge.svg)](https://app.codecov.io/gh/apache/arrow-julia) [![deps](https://juliahub.com/docs/Arrow/deps.svg)](https://juliahub.com/ui/Packages/Arrow/QnF3w?t=2) [![version](https://juliahub.com/docs/Arrow/version.svg)](https://juliahub.com/ui/Packages/Arrow/QnF3w) [![pkgeval](https://juliahub.com/docs/Arrow/pkgeval.svg)](https://juliahub.com/ui/Packages/Arrow/QnF3w) This is a pure Julia implementation of the [Apache Arrow](https://arrow.apache.org) data standard. This package provides Julia `AbstractVector` objects for referencing data that conforms to the Arrow standard. This allows users to seamlessly interface Arrow formatted data with a great deal of existing Julia code. Please see this [document](https://arrow.apache.org/docs/format/Columnar.html#physical-memory-layout) for a description of the Arrow memory layout. ## Installation The package can be installed by typing in the following in a Julia REPL: ```julia julia> using Pkg; Pkg.add("Arrow") ``` ## Local Development When developing on Arrow.jl it is recommended that you run the following to ensure that any changes to ArrowTypes.jl are immediately available to Arrow.jl without requiring a release: ```sh julia --project -e 'using Pkg; Pkg.develop(path="src/ArrowTypes")' ``` ## Format Support This implementation supports the 1.0 version of the specification, including support for: * All primitive data types * All nested data types * Dictionary encodings and messages * Extension types * Streaming, file, record batch, and replacement and isdelta dictionary messages It currently doesn't include support for: * Tensors or sparse tensors * Flight RPC * C data interface Third-party data formats: * CSV, parquet and avro support via the existing [CSV.jl](https://github.com/JuliaData/CSV.jl), [Parquet.jl](https://github.com/JuliaIO/Parquet.jl) and [Avro.jl](https://github.com/JuliaData/Avro.jl) packages * Other Tables.jl-compatible packages automatically supported ([DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), [JSONTables.jl](https://github.com/JuliaData/JSONTables.jl), [JuliaDB.jl](https://github.com/JuliaData/JuliaDB.jl), [SQLite.jl](https://github.com/JuliaDatabases/SQLite.jl), [MySQL.jl](https://github.com/JuliaDatabases/MySQL.jl), [JDBC.jl](https://github.com/JuliaDatabases/JDBC.jl), [ODBC.jl](https://github.com/JuliaDatabases/ODBC.jl), [XLSX.jl](https://github.com/felipenoris/XLSX.jl), etc.) * No current Julia packages support ORC See the [full documentation](https://arrow.apache.org/julia/) for details on reading and writing arrow data. ================================================ FILE: codecov.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. --- codecov: notify: # Wait for all "test" matrix jobs after_n_builds: 6 ================================================ FILE: dev/release/.dir-locals.el ================================================ ;;; Licensed to the Apache Software Foundation (ASF) under one ;;; or more contributor license agreements. See the NOTICE file ;;; distributed with this work for additional information ;;; regarding copyright ownership. The ASF licenses this file ;;; to you under the Apache License, Version 2.0 (the ;;; "License"); you may not use this file except in compliance ;;; with the License. You may obtain a copy of the License at ;;; ;;; http://www.apache.org/licenses/LICENSE-2.0 ;;; ;;; Unless required by applicable law or agreed to in writing, ;;; software distributed under the License is distributed on an ;;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ;;; KIND, either express or implied. See the License for the ;;; specific language governing permissions and limitations ;;; under the License. ((sh-mode . ((indent-tabs-mode . nil) (sh-basic-offset . 2)))) ================================================ FILE: dev/release/.gitignore ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. /apache-rat-*.jar /dist/ /filtered_rat.txt /rat.xml ================================================ FILE: dev/release/README.md ================================================ # Release ## Overview 1. Test the revision to be released 2. Increment version number in `Project.toml` 3. Prepare RC and vote (detailed later) 4. Publish (detailed later) ### Prepare RC and vote Run `dev/release/release_rc.sh` on working copy of `git@github.com:apache/arrow-julia` not your fork: ```console $ git clone git@github.com:apache/arrow-julia.git $ dev/release/release_rc.sh ${RC} (Send a vote email to dev@arrow.apache.org. You can use a draft shown by release_rc.sh for the email.) ``` Here is an example to release RC1: ```console $ dev/release/release_rc.sh 1 ``` The argument of `release_rc.sh` is the RC number. If RC1 has a problem, we'll increment the RC number such as RC2, RC3 and so on. Requirements to run `release_rc.sh`: * You must be an Apache Arrow committer or PMC member * You must prepare your PGP key for signing If you don't have a PGP key, https://infra.apache.org/release-signing.html#generate may be helpful. Your PGP key must be registered to the followings: * https://dist.apache.org/repos/dist/dev/arrow/KEYS * https://dist.apache.org/repos/dist/release/arrow/KEYS See the header comment of them how to add a PGP key. Apache arrow committers can update them by Subversion client with their ASF account. e.g.: ```console $ svn co https://dist.apache.org/repos/dist/dev/arrow $ cd arrow $ editor KEYS $ svn ci KEYS ``` ### Publish We need to do the followings to publish a new release: * Publish to apache.org * Publish to the Julia General registry Run `dev/release/release.sh` to publish to apache.org: ```console $ dev/release/release.sh ${VERSION} ${RC} ``` Here is an example to release 2.2.1 RC1: ```console $ dev/release/release.sh 2.2.1 1 ``` Add the release to ASF's report database via [Apache Committee Report Helper](https://reporter.apache.org/addrelease.html?arrow). To publish the release to the Julia General registry, navigate to the GitHub commit where the project version was incremented in the Project.toml file (step 2 above), then post a comment on the commit with the following: ```markdown @JuliaRegistrator register ``` JuliaRegistrator will respond saying it has opened a pull request to the General registry and under normal circumstances, will be merged automatically. If ArrowTypes is also registered, we also need to post a comment on the commit with the following: ```markdown @JuliaRegistrator register subdir=src/ArrowTypes ``` ### Verify We have a script to verify a RC. You must install the following commands to use the script: * `curl` * `gpg` * `shasum` or `sha256sum`/`sha512sum` * `tar` You don't need to install Julia. If there isn't Julia in system, the latest Julia is automatically installed only for verification. To verify a RC, run the following command line: ```console $ dev/release/verify_rc.sh ${VERSION} ${RC} ``` Here is an example to release 2.2.1 RC1: ```console $ dev/release/verify_rc.sh 2.2.1 1 ``` If the verification is succeeded, `RC looks good!` is shown. ================================================ FILE: dev/release/check_rat_report.py ================================================ #!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import fnmatch import re import sys import xml.etree.ElementTree as ET if len(sys.argv) != 3: sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % sys.argv[0]) sys.exit(1) exclude_globs_filename = sys.argv[1] xml_filename = sys.argv[2] globs = [line.strip() for line in open(exclude_globs_filename, "r")] tree = ET.parse(xml_filename) root = tree.getroot() resources = root.findall('resource') all_ok = True for r in resources: approvals = r.findall('license-approval') if not approvals or approvals[0].attrib['name'] == 'true': continue clean_name = re.sub('^[^/]+/', '', r.attrib['name']) excluded = False for g in globs: if fnmatch.fnmatch(clean_name, g): excluded = True break if not excluded: sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % ( clean_name, r.attrib['name'], approvals[0].attrib['name'])) all_ok = False if not all_ok: sys.exit(1) print('OK') sys.exit(0) ================================================ FILE: dev/release/rat_exclude_files.txt ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. Manifest.toml dev/release/apache-rat-*.jar dev/release/filtered_rat.txt dev/release/rat.xml test/arrowjson/*.json ================================================ FILE: dev/release/release.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. set -eu if [ "$#" -ne 2 ]; then echo "Usage: $0 " echo " e.g.: $0 2.2.1 1" exit 1 fi version=$1 rc=$2 rc_id="apache-arrow-julia-${version}-rc${rc}" release_id="arrow-julia-${version}" echo "Move from dev/ to release/" svn \ mv \ -m "Apache Arrow Julia ${version}" \ https://dist.apache.org/repos/dist/dev/arrow/${rc_id} \ https://dist.apache.org/repos/dist/release/arrow/${release_id} echo "Remove all RCs" old_rcs=$( svn ls https://dist.apache.org/repos/dist/dev/arrow/ | \ grep -E '^apache-arrow-julia-' | \ sort --version-sort --reverse ) for old_rc in $old_rcs; do echo "Remove RC ${old_rc}" svn \ delete \ -m "Remove old Apache Arrow Julia RC: ${old_rc}" \ https://dist.apache.org/repos/dist/dev/arrow/${old_rc} done echo "Keep only the latest versions" old_releases=$( svn ls https://dist.apache.org/repos/dist/release/arrow/ | \ grep -E '^arrow-julia-' | \ sort --version-sort --reverse | \ tail -n +2 ) for old_release_version in $old_releases; do echo "Remove old release ${old_release_version}" svn \ delete \ -m "Remove old Apache Arrow Julia release: ${old_release_version}" \ https://dist.apache.org/repos/dist/release/arrow/${old_release_version} done echo "Success! The release is available here:" echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" echo echo "Add this release to ASF's report database:" echo " https://reporter.apache.org/addrelease.html?arrow" ================================================ FILE: dev/release/release_rc.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. set -eu SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" if [ "$#" -ne 1 ]; then echo "Usage: $0 " echo " e.g.: $0 1" exit 1 fi rc=$1 : ${RELEASE_DEFAULT:=1} : ${RELEASE_CLEANUP:=${RELEASE_DEFAULT}} : ${RELEASE_PULL:=${RELEASE_DEFAULT}} : ${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}} : ${RELEASE_SIGN:=${RELEASE_DEFAULT}} : ${RELEASE_UPLOAD:=${RELEASE_DEFAULT}} cd "${SOURCE_TOP_DIR}" if [ ${RELEASE_PULL} -gt 0 -o ${RELEASE_PUSH_TAG} -gt 0 ]; then git_origin_url="$(git remote get-url origin)" if [ "${git_origin_url}" != "git@github.com:apache/arrow-julia.git" ]; then echo "This script must be ran with working copy of apache/arrow-julia." echo "The origin's URL: ${git_origin_url}" exit 1 fi fi if [ ${RELEASE_PULL} -gt 0 ]; then echo "Ensure using the latest commit" git checkout main git pull --rebase --prune fi version=$(grep -o '^version = ".*"' "Project.toml" | \ sed -e 's/^version = "//' \ -e 's/"$//') rc_tag="v${version}-rc${rc}" echo "Tagging for RC: ${rc_tag}" git tag -a -m "${version} RC${rc}" "${rc_tag}" if [ ${RELEASE_PUSH_TAG} -gt 0 ]; then git push origin "${rc_tag}" fi rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" id="apache-arrow-julia-${version}" rc_id="${id}-rc${rc}" dev_dist_url="https://dist.apache.org/repos/dist/dev/arrow" dev_dist_dir="dev/release/dist" tar_gz="${id}.tar.gz" tar_gz_path="${dev_dist_dir}/${rc_id}/${tar_gz}" rc_url="${dev_dist_url}/${rc_id}/" echo "Checking out ${dev_dist_url}" rm -rf "${dev_dist_dir}" svn co --depth=empty "${dev_dist_url}" "${dev_dist_dir}" echo "Attempting to create ${tar_gz} from tag ${rc_tag}" mkdir -p "$(dirname "${tar_gz_path}")" git archive "${rc_hash}" --prefix "${id}/" --output "${tar_gz_path}" pushd "${dev_dist_dir}/${rc_id}" echo "Running Rat license checker on ${tar_gz}" ../../run_rat.sh ${tar_gz} if [ ${RELEASE_SIGN} -gt 0 ]; then echo "Signing tar.gz and creating checksums" gpg --armor --output ${tar_gz}.asc --detach-sig ${tar_gz} fi if type shasum >/dev/null 2>&1; then sha256_generate="shasum -a 256" sha512_generate="shasum -a 512" else sha256_generate="sha256sum" sha512_generate="sha512sum" fi ${sha256_generate} ${tar_gz} > ${tar_gz}.sha256 ${sha512_generate} ${tar_gz} > ${tar_gz}.sha512 if [ ${RELEASE_UPLOAD} -gt 0 ]; then echo "Uploading to ${rc_url}" svn add . svn ci -m "Apache Arrow Julia ${version} ${rc}" fi popd if [ ${RELEASE_CLEANUP} -gt 0 ]; then echo "Removing temporary directory" rm -rf "${dev_dist_dir}" fi echo "Draft email for dev@arrow.apache.org mailing list" echo "" echo "---------------------------------------------------------" cat < "${RAT_XML}" FILTERED_RAT_TXT="${RELEASE_DIR}/filtered_rat.txt" if ${PYTHON:-python3} \ "${RELEASE_DIR}/check_rat_report.py" \ "${RELEASE_DIR}/rat_exclude_files.txt" \ "${RAT_XML}" > \ "${FILTERED_RAT_TXT}"; then echo "No unapproved licenses" else cat "${FILTERED_RAT_TXT}" N_UNAPPROVED=$(grep "NOT APPROVED" "${FILTERED_RAT_TXT}" | wc -l) echo "${N_UNAPPROVED} unapproved licenses. Check Rat report: ${RAT_XML}" exit 1 fi ================================================ FILE: dev/release/verify_rc.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. set -eu SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOP_SOURCE_DIR="$(dirname $(dirname ${SOURCE_DIR}))" if [ "$#" -ne 2 ]; then echo "Usage: $0 " echo " e.g.: $0 2.2.1 1" exit 1 fi set -o pipefail set -x VERSION="$1" RC="$2" ARROW_DIST_URL="https://dist.apache.org/repos/dist/dev/arrow" ARCHIVE_BASE_NAME="apache-arrow-julia-${VERSION}" : ${VERIFY_DEFAULT:=1} : ${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}} : ${VERIFY_FORCE_USE_JULIA_BINARY:=0} : ${VERIFY_SIGN:=${VERIFY_DEFAULT}} download_dist_file() { curl \ --fail \ --location \ --remote-name \ --show-error \ --silent \ "${ARROW_DIST_URL}/$1" } download_rc_file() { local path="apache-arrow-julia-${VERSION}-rc${RC}/$1" if [ ${VERIFY_DOWNLOAD} -gt 0 ]; then download_dist_file "${path}" else cp "${SOURCE_DIR}/dist/${path}" "$1" fi } import_gpg_keys() { if [ ${VERIFY_SIGN} -gt 0 ]; then download_dist_file KEYS gpg --import KEYS fi } if type shasum >/dev/null 2>&1; then sha256_verify="shasum -a 256 -c" sha512_verify="shasum -a 512 -c" else sha256_verify="sha256sum -c" sha512_verify="sha512sum -c" fi fetch_archive() { download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz if [ ${VERIFY_SIGN} -gt 0 ]; then download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.asc gpg --verify ${ARCHIVE_BASE_NAME}.tar.gz.asc ${ARCHIVE_BASE_NAME}.tar.gz fi download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.sha256 ${sha256_verify} ${ARCHIVE_BASE_NAME}.tar.gz.sha256 download_rc_file ${ARCHIVE_BASE_NAME}.tar.gz.sha512 ${sha512_verify} ${ARCHIVE_BASE_NAME}.tar.gz.sha512 } setup_tmpdir() { cleanup() { if [ "${VERIFY_SUCCESS}" = "yes" ]; then rm -rf "${VERIFY_TMPDIR}" else echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." fi } if [ -z "${VERIFY_TMPDIR:-}" ]; then VERIFY_TMPDIR=$(mktemp -d -t "$1.XXXXX") trap cleanup EXIT else mkdir -p "${VERIFY_TMPDIR}" fi } latest_julia_version() { curl \ --fail \ --location \ --show-error \ --silent \ https://api.github.com/repos/JuliaLang/julia/releases/latest | \ grep -o '"tag_name": "v.*"' | \ head -n 1 | \ sed -e 's/^"tag_name": "v//g' \ -e 's/"$//g' } ensure_julia() { if [ ${VERIFY_FORCE_USE_JULIA_BINARY} -le 0 ]; then if julia --version; then return fi fi local julia_binary_url=https://julialang-s3.julialang.org/bin local julia_version=$(latest_julia_version) local julia_version_series=${julia_version%.*} case "$(uname)" in Darwin) julia_binary_url+="/mac" case "$(arch)" in arm64) julia_binary_url+="/aarch64" julia_binary_url+="/${julia_version_series}" julia_binary_url+="/julia-${julia_version}-macaarch64.tar.gz" ;; i386) julia_binary_url+="/x64" julia_binary_url+="/${julia_version_series}" julia_binary_url+="/julia-${julia_version}-mac64.tar.gz" ;; *) echo "You must install Julia manually on $(uname) $(arch)" ;; esac ;; Linux) julia_binary_url+="/linux" case "$(arch)" in aarch64) julia_binary_url+="/aarch64" ;; x86_64) julia_binary_url+="/x64" ;; *) echo "You must install Julia manually on $(uname) $(arch)" ;; esac julia_binary_url+="/${julia_version_series}" julia_binary_url+="/julia-${julia_version}-linux-$(arch).tar.gz" ;; *) echo "You must install Julia manually on $(uname)" exit 1 ;; esac julia_binary_tar_gz=$(basename ${julia_binary_url}) curl \ --fail \ --location \ --output ${julia_binary_tar_gz} \ --show-error \ --silent \ ${julia_binary_url} tar xf ${julia_binary_tar_gz} julia_path=$(echo julia-*/bin/julia) PATH="$(pwd)/$(dirname ${julia_path}):${PATH}" export JULIA_DEPOT_PATH="$(pwd)/.julia" } test_source_distribution() { pushd src/ArrowTypes julia --project -e 'import Pkg; Pkg.build(); Pkg.test()' popd # Dev local ArrowTypes to use the version from this release, not from registry julia --project -e 'import Pkg; Pkg.develop(path="src/ArrowTypes"); Pkg.build(); Pkg.test()' } VERIFY_SUCCESS=no setup_tmpdir "arrow-julia-${VERSION}-${RC}" echo "Working in sandbox ${VERIFY_TMPDIR}" cd "${VERIFY_TMPDIR}" import_gpg_keys fetch_archive tar xf ${ARCHIVE_BASE_NAME}.tar.gz ensure_julia pushd ${ARCHIVE_BASE_NAME} test_source_distribution popd VERIFY_SUCCESS=yes echo "RC looks good!" ================================================ FILE: docs/.gitignore ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. build/ site/ ================================================ FILE: docs/Project.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [deps] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" [compat] Documenter = "1" ================================================ FILE: docs/make.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. using Documenter using Arrow makedocs(; modules=[Arrow], repo=Remotes.GitHub("apache", "arrow-julia"), sitename="Arrow.jl", format=Documenter.HTML(; prettyurls=get(ENV, "CI", "false") == "true", canonical="https://arrow.apache.org/julia/", assets=String[], ), pages=[ "Home" => "index.md", "User Manual" => "manual.md", "API Reference" => "reference.md", ], ) deploydocs(; repo="github.com/apache/arrow-julia", devbranch="main", branch="asf-site") ================================================ FILE: docs/src/index.md ================================================ ```@raw html ``` # Arrow.jl ```@contents Pages = ["manual.md", "reference.md"] Depth = 3 ``` ```@docs Arrow ``` ================================================ FILE: docs/src/manual.md ================================================ ```@raw html ``` # User Manual The goal of this documentation is to provide a brief introduction to the arrow data format, then provide a walk-through of the functionality provided in the Arrow.jl Julia package, with an aim to expose a little of the machinery "under the hood" to help explain how things work and how that influences real-world use-cases for the arrow data format. The best place to learn about the Apache arrow project is [the website itself](https://arrow.apache.org/), specifically the data format [specification](https://arrow.apache.org/docs/format/Columnar.html). Put briefly, the arrow project provides a formal specification for how columnar, "table" data can be laid out efficiently in memory to standardize and maximize the ability to share data across languages/platforms. In the current [apache/arrow GitHub repository](https://github.com/apache/arrow), language implementations exist for C++, Java, Go, Javascript, Rust, to name a few. Other database vendors and data processing frameworks/applications have also built support for the arrow format, allowing for a wide breadth of possibility for applications to "speak the data language" of arrow. The [Arrow.jl](https://github.com/apache/arrow-julia) Julia package is another implementation, allowing the ability to both read and write data in the arrow format. As a data format, arrow specifies an exact memory layout to be used for columnar table data, and as such, "reading" involves custom Julia objects ([`Arrow.Table`](@ref) and [`Arrow.Stream`](@ref)), which read the *metadata* of an "arrow memory blob", then *wrap* the array data contained therein, having learned the type and size, amongst other properties, from the metadata. Let's take a closer look at what this "reading" of arrow memory really means/looks like. ## Support for generic path-like types Arrow.jl attempts to support any path-like type wherever a function takes a path as an argument. The Arrow.jl API should generically work as long as the type supports: - `Base.open(path, mode)::I where I <: IO` When a custom `IO` subtype is returned (`I`) then the following methods also need to be defined: - `Base.read(io::I, ::Type{UInt8})` or `Base.read(io::I)` - `Base.write(io::I, x)` ## Reading arrow data After installing the Arrow.jl Julia package (via `] add Arrow`), and if you have some arrow data, let's say a file named `data.arrow` generated from the [`pyarrow`](https://arrow.apache.org/docs/python/) library (a Python library for interfacing with arrow data), you can then read that arrow data into a Julia session by doing: ```julia using Arrow table = Arrow.Table("data.arrow") ``` ### `Arrow.Table` The type of `table` in this example will be an `Arrow.Table`. When "reading" the arrow data, `Arrow.Table` first ["mmapped"](https://en.wikipedia.org/wiki/Mmap) the `data.arrow` file, which is an important technique for dealing with data larger than available RAM on a system. By "mmapping" a file, the OS doesn't actually load the entire file contents into RAM at the same time, but file contents are "swapped" into RAM as different regions of a file are requested. Once "mmapped", `Arrow.Table` then inspected the metadata in the file to determine the number of columns, their names and types, at which byte offset each column begins in the file data, and even how many "batches" are included in this file (arrow tables may be partitioned into one or more "record batches" each containing portions of the data). Armed with all the appropriate metadata, `Arrow.Table` then created custom array objects ([`Arrow.ArrowVector`](@ref)), which act as "views" into the raw arrow memory bytes. This is a significant point in that no extra memory is allocated for "data" when reading arrow data. This is in contrast to if we wanted to read data from a csv file as columns into Julia structures; we would need to allocate those array structures ourselves, then parse the file, "filling in" each element of the array with the data we parsed from the file. Arrow data, on the other hand, is *already laid out in memory or on disk* in a binary format, and as long as we have the metadata to interpret the raw bytes, we can figure out whether to treat those bytes as a `Vector{Float64}`, etc. A sample of the kinds of arrow array types you might see when deserializing arrow data, include: * [`Arrow.Primitive`](@ref): the most common array type for simple, fixed-size elements like integers, floats, time types, and decimals * [`Arrow.List`](@ref): an array type where its own elements are also arrays of some kind, like string columns, where each element can be thought of as an array of characters * [`Arrow.FixedSizeList`](@ref): similar to the `List` type, but where each array element has a fixed number of elements itself; you can think of this like a `Vector{NTuple{N, T}}`, where `N` is the fixed-size width * [`Arrow.Map`](@ref): an array type where each element is like a Julia `Dict`; a list of key value pairs like a `Vector{Dict}` * [`Arrow.Struct`](@ref): an array type where each element is an instance of a custom struct, i.e. an ordered collection of named & typed fields, kind of like a `Vector{NamedTuple}` * [`Arrow.DenseUnion`](@ref): an array type where elements may be of several different types, stored compactly; can be thought of like `Vector{Union{A, B}}` * [`Arrow.SparseUnion`](@ref): another array type where elements may be of several different types, but stored as if made up of identically lengthed child arrays for each possible type (less memory efficient than `DenseUnion`) * [`Arrow.DictEncoded`](@ref): a special array type where values are "dictionary encoded", meaning the list of unique, possible values for an array are stored internally in an "encoding pool", whereas each stored element of the array is just an integer "code" to index into the encoding pool for the actual value. And while these custom array types do subtype `AbstractArray`, there is no current support for `setindex!`. Remember, these arrays are "views" into the raw arrow bytes, so for array types other than `Arrow.Primitive`, it gets pretty tricky to allow manipulating those raw arrow bytes. Nevetheless, it's as simple as calling `copy(x)` where `x` is any `ArrowVector` type, and a normal Julia `Vector` type will be fully materialized (which would then allow mutating/manipulating values). So, what can you do with an `Arrow.Table` full of data? Quite a bit actually! Because `Arrow.Table` implements the [Tables.jl](https://juliadata.github.io/Tables.jl/stable/) interface, it opens up a world of integrations for using arrow data. A few examples include: * `df = DataFrame(Arrow.Table(file))`: Build a [`DataFrame`](https://juliadata.github.io/DataFrames.jl/stable/), using the arrow vectors themselves; this allows utilizing a host of DataFrames.jl functionality directly on arrow data; grouping, joining, selecting, etc. * `df = copy(DataFrame(Arrow.Table(file)))`: Build a [`DataFrame`](https://juliadata.github.io/DataFrames.jl/stable/), where the columns are regular in-memory vectors (specifically, `Base.Vector`s and/or `PooledVector`s). This requires that you have enough memory to load the entire `DataFrame` into memory. * `Tables.datavaluerows(Arrow.Table(file)) |> @map(...) |> @filter(...) |> DataFrame`: use [`Query.jl`'s](https://www.queryverse.org/Query.jl/stable/standalonequerycommands/) row-processing utilities to map, group, filter, mutate, etc. directly over arrow data. * `Arrow.Table(file) |> SQLite.load!(db, "arrow_table")`: load arrow data directly into an sqlite database/table, where sql queries can be executed on the data * `Arrow.Table(file) |> CSV.write("arrow.csv")`: write arrow data out to a csv file A full list of Julia packages leveraging the Tables.jl inteface can be found [here](https://github.com/JuliaData/Tables.jl/blob/master/INTEGRATIONS.md). Apart from letting other packages have all the fun, an `Arrow.Table` itself can be plenty useful. For example, with `tbl = Arrow.Table(file)`: * `tbl[1]`: retrieve the first column via indexing; the number of columns can be queried via `length(tbl)` * `tbl[:col1]` or `tbl.col1`: retrieve the column named `col1`, either via indexing with the column name given as a `Symbol`, or via "dot-access" * `for col in tbl`: iterate through columns in the table * `AbstractDict` methods like `haskey(tbl, :col1)`, `get(tbl, :col1, nothing)`, `keys(tbl)`, or `values(tbl)` ### Arrow types In the arrow data format, specific logical types are supported, a list of which can be found [here](https://arrow.apache.org/docs/status.html#data-types). These include booleans, integers of various bit widths, floats, decimals, time types, and binary/string. While most of these map naturally to types builtin to Julia itself, there are a few cases where the definitions are slightly different, and in these cases, by default, they are converted to more "friendly" Julia types (this auto conversion can be avoided by passing `convert=false` to `Arrow.Table`, like `Arrow.Table(file; convert=false)`). Examples of arrow to julia type mappings include: * `Date`, `Time`, `Timestamp`, and `Duration` all have natural Julia defintions in `Dates.Date`, `Dates.Time`, `TimeZones.ZonedDateTime`, and `Dates.Period` subtypes, respectively. * `Char` and `Symbol` Julia types are mapped to arrow string types, with additional metadata of the original Julia type; this allows deserializing directly to `Char` and `Symbol` in Julia, while other language implementations will see these columns as just strings * Similarly to the above, the `UUID` Julia type is mapped to a 128-bit `FixedSizeBinary` arrow type. * `Decimal128` and `Decimal256` have no corresponding builtin Julia types, so they're deserialized using a compatible type definition in Arrow.jl itself: `Arrow.Decimal` Note that when `convert=false` is passed, data will be returned in Arrow.jl-defined types that exactly match the arrow definitions of those types; the authoritative source for how each type represents its data can be found in the arrow [`Schema.fbs`](https://github.com/apache/arrow/blob/master/format/Schema.fbs) file. One note on performance: when writing `TimeZones.ZonedDateTime` columns to the arrow format (via `Arrow.write`), it is preferrable to "wrap" the columns in `Arrow.ToTimestamp(col)`, as long as the column has `ZonedDateTime` elements that all share a common timezone. This ensures the writing process can know "upfront" which timezone will be encoded and is thus much more efficient and performant. #### Custom types To support writing your custom Julia struct, Arrow.jl utilizes the format's mechanism for "extension types" by allowing the storing of Julia type name and metadata in the field metadata. To "hook in" to this machinery, custom types can utilize the interface methods defined in the `Arrow.ArrowTypes` submodule. For example: ```julia using Arrow struct Person id::Int name::String end # overload interface method for custom type Person; return a symbol as the "name" # this instructs Arrow.write what "label" to include with a column with this custom type const NAME = Symbol("JuliaLang.MyPackage.Person") ArrowTypes.arrowname(::Type{Person}) = NAME # overload JuliaType on `Val{NAME}`, which is like a dispatchable string # return our custom *type* Person; this enables Arrow.Table to know how the "label" # on a custom column should be mapped to a Julia type and deserialized ArrowTypes.JuliaType(::Val{NAME}) = Person table = (col1=[Person(1, "Bob"), Person(2, "Jane")],) io = IOBuffer() Arrow.write(io, table) seekstart(io) table2 = Arrow.Table(io) ``` In this example, we're writing our `table`, which is a NamedTuple with one column named `col1`, which has two elements which are instances of our custom `Person` struct. We overload `Arrowtypes.arrowname` so that Arrow.jl knows how to serialize our `Person` struct. We then overload `ArrowTypes.JuliaType` so the deserialization process knows how to map from our type label back to our `Person` struct type. We can then write our data in the arrow format to an in-memory `IOBuffer`, then read the table back in using `Arrow.Table`. The table we get back will be an `Arrow.Table`, with a single `Arrow.Struct` column with element type `Person`. Note that without calling `Arrowtypes.JuliaType`, we may get into a weird limbo state where we've written our table with `Person` structs out as a table, but when reading back in, Arrow.jl doesn't know what a `Person` is; deserialization won't fail, but we'll just get a `Namedtuple{(:id, :name), Tuple{Int, String}}` back instead of `Person`. While this example is very simple, it shows the basics to allow a custom type to be serialized/deserialized. But the `ArrowTypes` module offers even more powerful functionality for "hooking" non-native arrow types into the serialization/deserialization processes. Let's walk through a couple more examples; if you've had enough custom type shenanigans, feel free to skip to the next section. Let's take a look at how Arrow.jl allows serializing the `nothing` value, which is often referred to as the "software engineer's NULL" in Julia. While Arrow.jl treats `missing` as the default arrow NULL value, `nothing` is pretty similar, but we'd still like to treat it separately if possible. Here's how we enable serialization/deserialization in the `ArrowTypes` module: ```julia ArrowTypes.ArrowKind(::Type{Nothing}) = ArrowTypes.NullKind() ArrowTypes.ArrowType(::Type{Nothing}) = Missing ArrowTypes.toarrow(::Nothing) = missing const NOTHING = Symbol("JuliaLang.Nothing") ArrowTypes.arrowname(::Type{Nothing}) = NOTHING ArrowTypes.JuliaType(::Val{NOTHING}) = Nothing ArrowTypes.fromarrow(::Type{Nothing}, ::Missing) = nothing ``` Let's walk through what's going on here, line-by-line: * `ArrowKind` overload: `ArrowKind`s are generic "categories" of types supported by the arrow format, like `PrimitiveKind`, `ListKind`, etc. They each correspond to a different data layout strategy supported in the arrow format. Here, we define `nothing`'s kind to be `NullKind`, which means no actual memory is needed for storage, it's strictly a "metadata" type where we store the type and # of elements. In our `Person` example, we didn't need to overload this since types declared like `struct T` or `mutable struct T` are defined as `ArrowTypes.StructKind` by default * `ArrowType` overload: here we're signaling that our type (`Nothing`) maps to the natively supported arrow type of `Missing`; this is important for the serializer so it knows which arrow type it will be serializing. Again, we didn't need to overload this for `Person` since the serializer knows how to serialize custom structs automatically by using reflection methods like `fieldnames(T)` and `getfield(x, i)`. * `ArrowTypes.toarrow` overload: this is a sister method to `ArrowType`; we said our type will map to the `Missing` arrow type, so here we actually define ___how___ it converts to the arrow type; and in this case, it just returns `missing`. This is yet another method that didn't show up for `Person`; why? Well, as we noted in `ArrowType`, the serializer already knows how to serialize custom structs by using all their fields; if, for some reason, we wanted to omit some fields or otherwise transform things, then we could define corresponding `ArrowType` and `toarrow` methods * `arrowname` overload: similar to our `Person` example, we need to instruct the serializer how to label our custom type in the arrow type metadata; here we give it the symbol `Symbol("JuliaLang.Nothing")`. Note that while this will ultimately allow us to disambiguate `nothing` from `missing` when reading arrow data, if we pass this data to other language implementations, they will only treat the data as `missing` since they (probably) won't know how to "understand" the `JuliaLang.Nothing` type label * `JuliaType` overload: again, like our `Person` example, we instruct the deserializer that when it encounters the `JuliaLang.Nothing` type label, it should treat those values as `Nothing` type. * And finally, `fromarrow` overload: this allows specifying how the native-arrow data should be converted back to our custom type. `fromarrow(T, x...)` by default will call `T(x...)`, which is why we didn't need this overload for `Person`, but in this example, `Nothing(missing)` won't work, so we define our own custom conversion. Let's run through one more complex example, just for fun and to really see how far the system can be pushed: ```julia using Intervals table = (col = [ Interval{Closed,Unbounded}(1,nothing), ],) const NAME = Symbol("JuliaLang.Intervals.Interval") ArrowTypes.arrowname(::Type{Interval{T, L, R}}) where {T, L, R} = NAME const LOOKUP = Dict( "Closed" => Closed, "Unbounded" => Unbounded ) ArrowTypes.arrowmetadata(::Type{Interval{T, L, R}}) where {T, L, R} = string(L, ".", R) function ArrowTypes.JuliaType(::Val{NAME}, ::Type{NamedTuple{names, types}}, meta) where {names, types} L, R = split(meta, ".") return Interval{fieldtype(types, 1), LOOKUP[L], LOOKUP[R]} end ArrowTypes.fromarrow(::Type{Interval{T, L, R}}, first, last) where {T, L, R} = Interval{L, R}(first, R == Unbounded ? nothing : last) io = Arrow.tobuffer(table) tbl = Arrow.Table(io) ``` Again, let's break down what's going on here: * Here we're trying to save an `Interval` type in the arrow format; this type is unique in that it has two type parameters (`Closed` and `Unbounded`) that are not inferred/based on fields, but are just "type tags" on the type itself * Note that we define a generic `arrowname` method on all `Interval`s, regardless of type parameters. We just want to let arrow know which general type we're dealing with here * Next we use a new method `ArrowTypes.arrowmetadata` to encode the two non-field-based type parameters as a string with a dot delimiter; we encode this information here because remember, we have to match our `arrowname` Symbol typename in our `JuliaType(::Val(name))` definition in order to dispatch correctly; if we encoded the type parameters in `arrowname`, we would need separate `arrowname` definitions for each unique combination of those two type parameters, and corresponding `JuliaType` definitions for each as well; yuck. Instead, we let `arrowname` be generic to our type, and store the type parameters *for this specific column* using `arrowmetadata` * Now in `JuliaType`, note we're using the 3-argument overload; we want the `NamedTuple` type that is the native arrow type our `Interval` is being serialized as; we use this to retrieve the 1st type parameter for our `Interval`, which is simply the type of the two `first` and `last` fields. Then we use the 3rd argument, which is whatever string we returned from `arrowmetadata`. We call `L, R = split(meta, ".")` to parse the two type parameters (in this case `Closed` and `Unbounded`), then do a lookup on those strings from a predefined `LOOKUP` Dict that matches the type parameter name as string to the actual type. We then have all the information to recreate the full `Interval` type. Neat! * The one final wrinkle is in our `fromarrow` method; `Interval`s that are `Unbounded`, actually take `nothing` as the 2nd argument. So letting the default `fromarrow` definition call `Interval{T, L, R}(first, last)`, where `first` and `last` are both integers isn't going to work. Instead, we check if the `R` type parameter is `Unbounded` and if so, pass `nothing` as the 2nd arg, otherwise we can pass `last`. This stuff can definitely make your eyes glaze over if you stare at it long enough. As always, don't hesitate to reach out for quick questions on the [#data](https://julialang.slack.com/messages/data/) slack channel, or [open a new issue](https://github.com/apache/arrow-julia/issues/new) detailing what you're trying to do. ### `Arrow.Stream` In addition to `Arrow.Table`, the Arrow.jl package also provides `Arrow.Stream` for processing arrow data. While `Arrow.Table` will iterate all record batches in an arrow file/stream, concatenating columns, `Arrow.Stream` provides a way to *iterate* through record batches, one at a time. Each iteration yields an `Arrow.Table` instance, with columns/data for a single record batch. This allows, if so desired, "batch processing" of arrow data, one record batch at a time, instead of creating a single long table via `Arrow.Table`. ### Custom application metadata The Arrow format allows data producers to [attach custom metadata](https://arrow.apache.org/docs/format/Columnar.html#custom-application-metadata) to various Arrow objects. Arrow.jl provides a convenient accessor for this metadata via [`Arrow.getmetadata`](@ref). `Arrow.getmetadata(t::Arrow.Table)` will return an immutable `AbstractDict{String,String}` that represents the [`custom_metadata` of the table's associated `Schema`](https://github.com/apache/arrow/blob/85d8175ea24b4dd99f108a673e9b63996d4f88cc/format/Schema.fbs#L515) (or `nothing` if no such metadata exists), while `Arrow.getmetadata(c::Arrow.ArrowVector)` will return a similar representation of [the column's associated `Field` `custom_metadata`](https://github.com/apache/arrow/blob/85d8175ea24b4dd99f108a673e9b63996d4f88cc/format/Schema.fbs#L480) (or `nothing` if no such metadata exists). To attach custom schema/column metadata to Arrow tables at serialization time, see the `metadata` and `colmetadata` keyword arguments to [`Arrow.write`](@ref). ## Writing arrow data Ok, so that's a pretty good rundown of *reading* arrow data, but how do you *produce* arrow data? Enter `Arrow.write`. ### `Arrow.write` With `Arrow.write`, you provide either an `io::IO` argument or a [`file_path`](#support-for-generic-path-like-types) to write the arrow data to, as well as a Tables.jl-compatible source that contains the data to be written. What are some examples of Tables.jl-compatible sources? A few examples include: * `Arrow.write(io, df::DataFrame)`: A `DataFrame` is a collection of indexable columns * `Arrow.write(io, CSV.File(file))`: read data from a csv file and write out to arrow format * `Arrow.write(io, DBInterface.execute(db, sql_query))`: Execute an SQL query against a database via the [`DBInterface.jl`](https://github.com/JuliaDatabases/DBInterface.jl) interface, and write the query resultset out directly in the arrow format. Packages that implement DBInterface include [SQLite.jl](https://juliadatabases.github.io/SQLite.jl/stable/), [MySQL.jl](https://juliadatabases.github.io/MySQL.jl/dev/), and [ODBC.jl](http://juliadatabases.github.io/ODBC.jl/latest/). * `df |> @map(...) |> Arrow.write(io)`: Write the results of a [Query.jl](https://www.queryverse.org/Query.jl/stable/) chain of operations directly out as arrow data * `jsontable(json) |> Arrow.write(io)`: Treat a json array of objects or object of arrays as a "table" and write it out as arrow data using the [JSONTables.jl](https://github.com/JuliaData/JSONTables.jl) package * `Arrow.write(io, (col1=data1, col2=data2, ...))`: a `NamedTuple` of `AbstractVector`s or an `AbstractVector` of `NamedTuple`s are both considered tables by default, so they can be quickly constructed for easy writing of arrow data if you already have columns of data And these are just a few examples of the numerous [integrations](https://github.com/JuliaData/Tables.jl/blob/master/INTEGRATIONS.md). In addition to just writing out a single "table" of data as a single arrow record batch, `Arrow.write` also supports writing out multiple record batches when the input supports the `Tables.partitions` functionality. One immediate, though perhaps not incredibly useful example, is `Arrow.Stream`. `Arrow.Stream` implements `Tables.partitions` in that it iterates "tables" (specifically `Arrow.Table`), and as such, `Arrow.write` will iterate an `Arrow.Stream`, and write out each `Arrow.Table` as a separate record batch. Another important point for why this example works is because an `Arrow.Stream` iterates `Arrow.Table`s that all have the same schema. This is important because when writing arrow data, a "schema" message is always written first, with all subsequent record batches written with data matching the initial schema. In addition to inputs that support `Tables.partitions`, note that the Tables.jl itself provides the `Tables.partitioner` function, which allows providing your own separate instances of similarly-schema-ed tables as "partitions", like: ```julia # treat 2 separate NamedTuples of vectors with same schema as 1 table, 2 partitions tbl_parts = Tables.partitioner([(col1=data1, col2=data2), (col1=data3, col2=data4)]) Arrow.write(io, tbl_parts) # treat an array of csv files with same schema where each file is a partition # in this form, a function `CSV.File` is applied to each element of 2nd argument csv_parts = Tables.partitioner(CSV.File, csv_files) Arrow.write(io, csv_parts) ``` ### `Arrow.Writer` With `Arrow.Writer`, you instantiate an `Arrow.Writer` object, write sources using it, and then close it. This allows for incrmental writes to the same sink. It is similar to `Arrow.append` without having to close and re-open the sink in between writes and without the limitation of only supporting the IPC stream format. ### Multithreaded writing By default, `Arrow.write` will use multiple threads to write multiple record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set). The number of concurrent tasks to use when writing can be controlled by passing the `ntasks` keyword argument to `Arrow.write`. Passing `ntasks=1` avoids any multithreading when writing. ### Compression Compression is supported when writing via the `compress` keyword argument. Possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding or compressor. ================================================ FILE: docs/src/reference.md ================================================ ```@raw html ``` # API Reference ```@autodocs Modules = [Arrow] Order = [:type, :function] ``` ## Internals: `Arrow.FlatBuffers` The `FlatBuffers` module is not part of Arrow.jl's public API, and these functions may change without notice. ```@autodocs Modules = [Arrow.FlatBuffers] ``` ================================================ FILE: src/Arrow.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.jl A pure Julia implementation of the [apache arrow](https://arrow.apache.org/) memory format specification. This implementation supports the 1.0 version of the specification, including support for: * All primitive data types * All nested data types * Dictionary encodings, nested dictionary encodings, and messages * Extension types * Streaming, file, record batch, and replacement and isdelta dictionary messages * Buffer compression/decompression via the standard LZ4 frame and Zstd formats It currently doesn't include support for: * Tensors or sparse tensors * Flight RPC * C data interface Third-party data formats: * csv and parquet support via the existing [CSV.jl](https://github.com/JuliaData/CSV.jl) and [Parquet.jl](https://github.com/JuliaIO/Parquet.jl) packages * Other [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible packages automatically supported ([DataFrames.jl](https://github.com/JuliaData/DataFrames.jl), [JSONTables.jl](https://github.com/JuliaData/JSONTables.jl), [JuliaDB.jl](https://github.com/JuliaData/JuliaDB.jl), [SQLite.jl](https://github.com/JuliaDatabases/SQLite.jl), [MySQL.jl](https://github.com/JuliaDatabases/MySQL.jl), [JDBC.jl](https://github.com/JuliaDatabases/JDBC.jl), [ODBC.jl](https://github.com/JuliaDatabases/ODBC.jl), [XLSX.jl](https://github.com/felipenoris/XLSX.jl), etc.) * No current Julia packages support ORC or Avro data formats See docs for official Arrow.jl API with the [User Manual](@ref) and reference docs for [`Arrow.Table`](@ref), [`Arrow.write`](@ref), and [`Arrow.Stream`](@ref). """ module Arrow using Base.Iterators using Mmap import Dates using DataAPI, Tables, SentinelArrays, PooledArrays, CodecLz4, CodecZstd, TimeZones, BitIntegers, ConcurrentUtilities, StringViews export ArrowTypes using Base: @propagate_inbounds import Base: == const FILE_FORMAT_MAGIC_BYTES = b"ARROW1" const CONTINUATION_INDICATOR_BYTES = 0xffffffff # vendored flatbuffers code for now include("FlatBuffers/FlatBuffers.jl") using .FlatBuffers include("metadata/Flatbuf.jl") using .Flatbuf const Meta = Flatbuf using ArrowTypes include("utils.jl") include("arraytypes/arraytypes.jl") include("eltypes.jl") include("table.jl") include("write.jl") include("append.jl") include("show.jl") const ZSTD_COMPRESSOR = Lockable{ZstdCompressor}[] const ZSTD_DECOMPRESSOR = Lockable{ZstdDecompressor}[] const LZ4_FRAME_COMPRESSOR = Lockable{LZ4FrameCompressor}[] const LZ4_FRAME_DECOMPRESSOR = Lockable{LZ4FrameDecompressor}[] function init_zstd_compressor() zstd = ZstdCompressor(; level=3) CodecZstd.TranscodingStreams.initialize(zstd) return Lockable(zstd) end function init_zstd_decompressor() zstd = ZstdDecompressor() CodecZstd.TranscodingStreams.initialize(zstd) return Lockable(zstd) end function init_lz4_frame_compressor() lz4 = LZ4FrameCompressor(; compressionlevel=4) CodecLz4.TranscodingStreams.initialize(lz4) return Lockable(lz4) end function init_lz4_frame_decompressor() lz4 = LZ4FrameDecompressor() CodecLz4.TranscodingStreams.initialize(lz4) return Lockable(lz4) end function access_threaded(f, v::Vector) tid = Threads.threadid() 0 < tid <= length(v) || _length_assert() if @inbounds isassigned(v, tid) @inbounds x = v[tid] else x = f() @inbounds v[tid] = x end return x end @noinline _length_assert() = @assert false "0 < tid <= v" zstd_compressor() = access_threaded(init_zstd_compressor, ZSTD_COMPRESSOR) zstd_decompressor() = access_threaded(init_zstd_decompressor, ZSTD_DECOMPRESSOR) lz4_frame_compressor() = access_threaded(init_lz4_frame_compressor, LZ4_FRAME_COMPRESSOR) lz4_frame_decompressor() = access_threaded(init_lz4_frame_decompressor, LZ4_FRAME_DECOMPRESSOR) function __init__() nt = @static if isdefined(Base.Threads, :maxthreadid) Threads.maxthreadid() else Threads.nthreads() end resize!(empty!(LZ4_FRAME_COMPRESSOR), nt) resize!(empty!(ZSTD_COMPRESSOR), nt) resize!(empty!(LZ4_FRAME_DECOMPRESSOR), nt) resize!(empty!(ZSTD_DECOMPRESSOR), nt) return end end # module Arrow ================================================ FILE: src/ArrowTypes/LICENSE.md ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: src/ArrowTypes/Project.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name = "ArrowTypes" uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd" authors = ["quinnj "] version = "2.3.0" [deps] Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] julia = "1.0" ================================================ FILE: src/ArrowTypes/src/ArrowTypes.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ The ArrowTypes module provides the [`ArrowTypes.Arrowtype`](@ref) interface trait that objects can define in order to signal how they should be serialized in the arrow format. """ module ArrowTypes using Sockets using UUIDs export ArrowKind, NullKind, PrimitiveKind, BoolKind, ListKind, FixedSizeListKind, MapKind, StructKind, UnionKind, DictEncodedKind, toarrow, arrowname, fromarrow, ToArrow """ ArrowTypes.ArrowKind(T) For a give type `T`, define it's "arrow type kind", or the general category of arrow types it should be treated as. Must be one of: * [`ArrowTypes.NullKind`](@ref): `Missing` is the only type defined as `NullKind` * [`ArrowTypes.PrimitiveKind`](@ref): `<:Integer`, `<:AbstractFloat`, along with `Arrow.Decimal`, and the various `Arrow.ArrowTimeType` subtypes * [`ArrowTypes.BoolKind`](@ref): only `Bool` * [`ArrowTypes.ListKind`](@ref): any `AbstractString` or `AbstractArray` * [`ArrowTypes.FixedSizeList`](@ref): `NTuple{N, T}` * [`ArrowTypes.MapKind`](@ref): any `AbstractDict` * [`ArrowTypes.StructKind`](@ref): any `NamedTuple` or plain struct (mutable or otherwise) * [`ArrowTypes.UnionKind`](@ref): any `Union` * [`ArrowTypes.DictEncodedKind`](@ref): array types that implement the `DataAPI.refpool` interface The list of `ArrowKind`s listed above translate to different ways to physically store data as supported by the arrow data format. See the docs for each for an idea of whether they might be an appropriate fit for a custom type. Note that custom types need to satisfy any additional "interface methods" as required by the various `ArrowKind` types. By default, if a type in julia is declared like `primitive type ...` it is considered a `PrimitiveKind` and if `struct` or `mutable struct` it's considered a `StructKind`. Also note that types will rarely need to define `ArrowKind`; much more common is to define `ArrowType(T)` and `toarrow(x::T)` to transform `T` to a natively supported arrow type, which will already have its `ArrowKind` defined. """ abstract type ArrowKind end ArrowKind(x::T) where {T} = ArrowKind(T) ArrowKind(::Type{T}) where {T} = isprimitivetype(T) ? PrimitiveKind() : StructKind() """ ArrowTypes.ArrowType(T) = S Interface method to define the natively supported arrow type `S` that a given type `T` should be converted to before serializing. Useful when a custom type wants a "serialization hook" or otherwise needs to be transformed/converted into a natively supported arrow type for serialization. If a type defines `ArrowType`, it must also define a corresponding [`ArrowTypes.toarrow(x::T)`](@ref) method which does the actual conversion from `T` to `S`. Note that custom structs defined like `struct T` or `mutable struct T` are natively supported in serialization, so unless _additional_ transformation/customization is desired, a custom type `T` can serialize with no `ArrowType` definition (by default, each field of a struct is serialized, using the results of `fieldnames(T)` and `getfield(x, i)`). Note that defining these methods only deal with custom _serialization_ to the arrow format; to be able to _deserialize_ custom types at all, see the docs for [`ArrowTypes.arrowname`](@ref), [`ArrowTypes.arrowmetadata`](@ref), [`ArrowTypes.JuliaType`](@ref), and [`ArrowTypes.fromarrow`](@ref). """ function ArrowType end ArrowType(::Type{T}) where {T} = T ArrowType(::Type{Any}) = Any ArrowType(::Type{Union{Missing,T}}) where {T} = Union{Missing,ArrowType(T)} ArrowType(::Type{Missing}) = Missing """ ArrowTypes.toarrow(x::T) => S Interface method to perform the actual conversion from an object `x` of type `T` to the type `S`. `T` and `S` must match the types used when defining `ArrowTypes.ArrowType(::Type{T}) = S`. Hence, `S` is the natively supported arrow type that `T` desires to convert to to enable serialization. See [`ArrowTypes.ArrowType`](@ref) docs for more details. This enables custom objects to be serialized as a natively supported arrow data type. """ function toarrow end toarrow(x) = x """ ArrowTypes.arrowname(T) = Symbol(name) Interface method to define the logical type "label" for a custom Julia type `T`. Names will be global for an entire arrow dataset, and conventionally, custom types will just use their type name along with a Julia- and package-specific prefix; for example, for a custom type `Foo`, I would define `ArrowTypes.arrowname(::Type{Foo}) = Symbol("JuliaLang.MyPackage.Foo")`. This ensures other language implementations won't get confused and are safe to ignore the logical type label. When arrow stores non-native data, it must still be _stored_ as a native data type, but can have type metadata tied to the data that labels the original _logical_ type it originated from. This enables the conversion of native data back to the logical type when deserializing, as long as the deserializer has the same definitions when the data was serialized. Namely, the current Julia session will need the appropriate [`ArrowTypes.JuliaType`](@ref) and [`ArrowTypes.fromarrow`](@ref) definitions in order to know how to convert the native data to the original logical type. See the docs for those interface methods in order to ensure a complete implementation. Also see the accompanying [`ArrowTypes.arrowmetadata`](@ref) docs around providing additional metadata about a custom logical type that may be necessary to fully re-create a Julia type (e.g. non-field-based type parameters). """ function arrowname end const EMPTY_SYMBOL = Symbol() arrowname(T) = EMPTY_SYMBOL hasarrowname(T) = arrowname(T) !== EMPTY_SYMBOL arrowname(::Type{Union{T,Missing}}) where {T} = arrowname(T) arrowname(::Type{Union{T,Nothing}}) where {T} = arrowname(T) arrowname(::Type{Missing}) = EMPTY_SYMBOL arrowname(::Type{Any}) = EMPTY_SYMBOL """ ArrowTypes.arrowmetadata(T) => String Interface method to provide additional logical type metadata when serializing extension types. [`ArrowTypes.arrowname`](@ref) provides the logical type _name_, which may be all that's needed to return a proper Julia type from [`ArrowTypes.JuliaType`](@ref), but some custom types may, for example have type parameters that aren't inferred/based on fields. In order to fully recreate these kinds of types when deserializing, these type parameters can be stored by defining `ArrowTypes.arrowmetadata(::Type{T}) = "type_param"`. This will then be available to access by overloading `ArrowTypes.JuliaType(::Val{Symbol(name)}, S, arrowmetadata::String)`. """ function arrowmetadata end const EMPTY_STRING = "" arrowmetadata(T) = EMPTY_STRING arrowmetadata(::Type{Union{T,Missing}}) where {T} = arrowmetadata(T) arrowmetadata(::Type{Union{T,Nothing}}) where {T} = arrowmetadata(T) arrowmetadata(::Type{Nothing}) = EMPTY_STRING arrowmetadata(::Type{Missing}) = EMPTY_STRING arrowmetadata(::Type{Any}) = EMPTY_STRING """ ArrowTypes.JuliaType(::Val{Symbol(name)}, ::Type{S}, arrowmetadata::String) = T Interface method to define the custom Julia logical type `T` that a serialized metadata label should be converted to when deserializing. When reading arrow data, and a logical type label is encountered for a column, it will call `ArrowTypes.JuliaType(Val(Symbol(name)), S, arrowmetadata)` to see if a Julia type has been "registered" for deserialization. The `name` used when defining the method *must* correspond to the same `name` when defining `ArrowTypes.arrowname(::Type{T}) = Symbol(name)`. The use of `Val(Symbol(...))` is to allow overloading a method on a specific logical type label. The `S` 2nd argument passed to `JuliaType` is the native arrow serialized type. This can be useful for parametric Julia types that wish to correctly parameterize their custom type based on what was serialized. The 3rd argument `arrowmetadata` is any metadata that was stored when the logical type was serialized as the result of calling `ArrowTypes.arrowmetadata(T)`. Note the 2nd and 3rd arguments are optional when overloading if unneeded. When defining [`ArrowTypes.arrowname`](@ref) and `ArrowTypes.JuliaType`, you may also want to implement [`ArrowTypes.fromarrow`] in order to customize how a custom type `T` should be constructed from the native arrow data type. See its docs for more details. """ function JuliaType end JuliaType(val) = nothing JuliaType(val, S) = JuliaType(val) JuliaType(val, S, meta) = JuliaType(val, S) """ ArrowTypes.fromarrow(::Type{T}, x::S) => T Interface method that provides a "deserialization hook" for a custom type `T` to be constructed from the native arrow type `S`. The `T` and `S` types must correspond to the definitions used in `ArrowTypes.ArrowType(::Type{T}) = S`. This is a paired method with [`ArrowTypes.toarrow`](@ref). The default definition is `ArrowTypes.fromarrow(::Type{T}, x) = T(x)`, so if that works for a custom type already, no additional overload is necessary. A few `ArrowKind`s have/allow slightly more custom overloads for their `fromarrow` methods: * `ListKind{true}`: for `String` types, they may overload `fromarrow(::Type{T}, ptr::Ptr{UInt8}, len::Int) = ...` to avoid materializing a `String` * `StructKind`: * May overload `fromarrow(::Type{T}, x...)` where individual fields are passed as separate positional arguments; so if my custom type `Interval` has two fields `first` and `last`, then I'd overload like `ArrowTypes.fromarrow(::Type{Interval}, first, last) = ...`. Note the default implementation is `ArrowTypes.fromarrow(::Type{T}, x...) = T(x...)`, so if your type already accepts all arguments in a constructor no additional `fromarrow` method should be necessary (default struct constructors have this behavior). * Alternatively, may overload `fromarrowstruct(::Type{T}, ::Val{fnames}, x...)`, where `fnames` is a tuple of the field names corresponding to the values in `x`. This approach is useful when you need to implement deserialization in a manner that is agnostic to the field order used by the serializer. When implemented, `fromarrowstruct` takes precedence over `fromarrow` in `StructKind` deserialization. """ function fromarrow end fromarrow(::Type{T}, x::T) where {T} = x fromarrow(::Type{T}, x...) where {T} = T(x...) fromarrow(::Type{Union{Missing,T}}, ::Missing) where {T} = missing fromarrow(::Type{Union{Missing,T}}, x::T) where {T} = x fromarrow(::Type{Union{Missing,T}}, x::T) where {T<:NamedTuple} = x # ambiguity fix fromarrow(::Type{Union{Missing,T}}, x) where {T} = fromarrow(T, x) "NullKind data is actually not physically stored since the data is constant; just the length is needed" struct NullKind <: ArrowKind end ArrowKind(::Type{Missing}) = NullKind() ArrowKind(::Type{Nothing}) = NullKind() ArrowType(::Type{Nothing}) = Missing toarrow(::Nothing) = missing const NOTHING = Symbol("JuliaLang.Nothing") arrowname(::Type{Nothing}) = NOTHING JuliaType(::Val{NOTHING}) = Nothing fromarrow(::Type{Nothing}, ::Missing) = nothing "PrimitiveKind data is stored as plain bits in a single contiguous buffer" struct PrimitiveKind <: ArrowKind end ArrowKind(::Type{<:Integer}) = PrimitiveKind() ArrowKind(::Type{<:AbstractFloat}) = PrimitiveKind() ArrowType(::Type{Char}) = UInt32 toarrow(x::Char) = convert(UInt32, x) const CHAR = Symbol("JuliaLang.Char") arrowname(::Type{Char}) = CHAR JuliaType(::Val{CHAR}) = Char fromarrow(::Type{Char}, x::UInt32) = Char(x) "BoolKind data is stored with values packed down to individual bits; so instead of a traditional Bool being 1 byte/8 bits, 8 Bool values would be packed into a single byte" struct BoolKind <: ArrowKind end ArrowKind(::Type{Bool}) = BoolKind() "ListKind data are stored in two separate buffers; one buffer contains all the original data elements flattened into one long buffer; the 2nd buffer contains an offset into the 1st buffer for how many elements make up the original array element" struct ListKind{stringtype} <: ArrowKind end ListKind() = ListKind{false}() isstringtype(::ListKind{stringtype}) where {stringtype} = stringtype isstringtype(::Type{ListKind{stringtype}}) where {stringtype} = stringtype ArrowKind(::Type{<:AbstractString}) = ListKind{true}() # Treate Base.CodeUnits as Binary arrow type ArrowKind(::Type{<:Base.CodeUnits}) = ListKind{true}() fromarrow(::Type{T}, ptr::Ptr{UInt8}, len::Int) where {T} = fromarrow(T, unsafe_string(ptr, len)) fromarrow(::Type{T}, x) where {T<:Base.CodeUnits} = Base.CodeUnits(x) fromarrow(::Type{Union{Missing,Base.CodeUnits}}, x) = x === missing ? missing : Base.CodeUnits(x) ArrowType(::Type{Symbol}) = String toarrow(x::Symbol) = String(x) const SYMBOL = Symbol("JuliaLang.Symbol") arrowname(::Type{Symbol}) = SYMBOL JuliaType(::Val{SYMBOL}) = Symbol _symbol(ptr, len) = ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int), ptr, len) fromarrow(::Type{Symbol}, ptr::Ptr{UInt8}, len::Int) = _symbol(ptr, len) ArrowKind(::Type{<:AbstractArray}) = ListKind() fromarrow(::Type{A}, x::A) where {A<:AbstractVector{T}} where {T} = x fromarrow(::Type{A}, x::AbstractVector{T}) where {A<:AbstractVector{T}} where {T} = convert(A, x) ArrowKind(::Type{<:AbstractSet}) = ListKind() ArrowType(::Type{T}) where {T<:AbstractSet{S}} where {S} = Vector{S} toarrow(x::AbstractSet) = collect(x) const SET = Symbol("JuliaLang.Set") arrowname(::Type{<:AbstractSet}) = SET JuliaType(::Val{SET}, ::Type{T}) where {T<:AbstractVector{S}} where {S} = Set{S} fromarrow(::Type{T}, x) where {T<:AbstractSet} = T(x) "FixedSizeListKind data are stored in a single contiguous buffer; individual elements can be computed based on the fixed size of the lists" struct FixedSizeListKind{N,T} <: ArrowKind end gettype(::FixedSizeListKind{N,T}) where {N,T} = T getsize(::FixedSizeListKind{N,T}) where {N,T} = N ArrowKind(::Type{NTuple{N,T}}) where {N,T} = FixedSizeListKind{N,T}() ArrowKind(::Type{UUID}) = FixedSizeListKind{16,UInt8}() ArrowType(::Type{UUID}) = NTuple{16,UInt8} toarrow(x::UUID) = _cast(NTuple{16,UInt8}, x.value) const UUIDSYMBOL = Symbol("JuliaLang.UUID") arrowname(::Type{UUID}) = UUIDSYMBOL JuliaType(::Val{UUIDSYMBOL}) = UUID fromarrow(::Type{UUID}, x::NTuple{16,UInt8}) = UUID(_cast(UInt128, x)) ArrowKind(::Type{IPv4}) = PrimitiveKind() ArrowType(::Type{IPv4}) = UInt32 toarrow(x::IPv4) = x.host const IPV4_SYMBOL = Symbol("JuliaLang.IPv4") arrowname(::Type{IPv4}) = IPV4_SYMBOL JuliaType(::Val{IPV4_SYMBOL}) = IPv4 fromarrow(::Type{IPv4}, x::Integer) = IPv4(x) ArrowKind(::Type{IPv6}) = FixedSizeListKind{16,UInt8}() ArrowType(::Type{IPv6}) = NTuple{16,UInt8} toarrow(x::IPv6) = _cast(NTuple{16,UInt8}, x.host) const IPV6_SYMBOL = Symbol("JuliaLang.IPv6") arrowname(::Type{IPv6}) = IPV6_SYMBOL JuliaType(::Val{IPV6_SYMBOL}) = IPv6 fromarrow(::Type{IPv6}, x::NTuple{16,UInt8}) = IPv6(_cast(UInt128, x)) function _cast(::Type{Y}, x)::Y where {Y} y = Ref{Y}() _unsafe_cast!(y, Ref(x), 1) return y[] end function _unsafe_cast!(y::Ref{Y}, x::Ref, n::Integer) where {Y} X = eltype(x) GC.@preserve x y begin ptr_x = Base.unsafe_convert(Ptr{X}, x) ptr_y = Base.unsafe_convert(Ptr{Y}, y) unsafe_copyto!(Ptr{X}(ptr_y), ptr_x, n) end return y end "StructKind data are stored in separate buffers for each field of the struct" struct StructKind <: ArrowKind end ArrowKind(::Type{<:NamedTuple}) = StructKind() @inline fromarrowstruct(T::Type, ::Val, x...) = fromarrow(T, x...) fromarrow( ::Type{NamedTuple{names,types}}, x::NamedTuple{names,types}, ) where {names,types<:Tuple} = x fromarrow(::Type{T}, x::NamedTuple) where {T} = fromarrow(T, Tuple(x)...) ArrowKind(::Type{<:Tuple}) = StructKind() ArrowKind(::Type{Tuple{}}) = StructKind() const TUPLE = Symbol("JuliaLang.Tuple") # needed to disambiguate the FixedSizeList case for NTuple arrowname(::Type{NTuple{N,T}}) where {N,T} = EMPTY_SYMBOL arrowname(::Type{T}) where {T<:Tuple} = TUPLE arrowname(::Type{Tuple{}}) = TUPLE JuliaType(::Val{TUPLE}, ::Type{NamedTuple{names,types}}) where {names,types<:Tuple} = types fromarrow(::Type{T}, x::NamedTuple) where {T<:Tuple} = Tuple(x) # VersionNumber const VERSION_NUMBER = Symbol("JuliaLang.VersionNumber") ArrowKind(::Type{VersionNumber}) = StructKind() arrowname(::Type{VersionNumber}) = VERSION_NUMBER JuliaType(::Val{VERSION_NUMBER}) = VersionNumber default(::Type{VersionNumber}) = v"0" function fromarrow(::Type{VersionNumber}, v::NamedTuple) VersionNumber(v.major, v.minor, v.patch, v.prerelease, v.build) end "MapKind data are stored similarly to ListKind, where elements are flattened, and a 2nd offsets buffer contains the individual list element length data" struct MapKind <: ArrowKind end ArrowKind(::Type{<:AbstractDict}) = MapKind() "UnionKind data are stored either in a separate, compacted buffer for each union type (dense), or in full-length buffers for each union type (sparse)" struct UnionKind <: ArrowKind end ArrowKind(::Union) = UnionKind() "DictEncodedKind store a small pool of unique values in one buffer, with a full-length buffer of integer offsets into the small value pool" struct DictEncodedKind <: ArrowKind end """ There are a couple places when writing arrow buffers where we need to write a "dummy" value; it doesn't really matter what we write, but we need to write something of a specific type. So each supported writing type needs to define `default`. """ function default end default(T) = zero(T) default(::Type{Symbol}) = Symbol() default(::Type{Char}) = '\0' default(::Type{<:AbstractString}) = "" default(::Type{Any}) = nothing default(::Type{Missing}) = missing default(::Type{Nothing}) = nothing default(::Type{Union{T,Missing}}) where {T} = default(T) default(::Type{Union{T,Nothing}}) where {T} = default(T) default(::Type{Union{T,Missing,Nothing}}) where {T} = default(T) function default(::Type{A}) where {A<:AbstractVector{T}} where {T} a = similar(A, 1) a[1] = default(T) return a end default(::Type{SubArray{T,N,P,I,L}}) where {T,N,P,I,L} = view(default(P), 0:-1) default(::Type{NTuple{N,T}}) where {N,T} = ntuple(i -> default(T), N) default(::Type{Tuple{}}) = () function default(::Type{T}) where {T<:Tuple} T === Tuple{} && return () N = Base.isvarargtype(T.parameters[end]) ? length(T.parameters) - 1 : fieldcount(T) return Tuple(default(fieldtype(T, i)) for i = 1:N) end default(::Type{T}) where {T<:AbstractDict} = T() default(::Type{NamedTuple{names,types}}) where {names,types} = NamedTuple{names}(Tuple(default(fieldtype(types, i)) for i = 1:length(names))) function promoteunion(T, S) new = promote_type(T, S) return isabstracttype(new) ? Union{T,S} : new end # lazily call toarrow(x) on getindex for each x in data struct ToArrow{T,A} <: AbstractVector{T} data::A end concrete_or_concreteunion(T) = isconcretetype(T) || (T isa Union && concrete_or_concreteunion(T.a) && concrete_or_concreteunion(T.b)) function ToArrow(x::A) where {A} S = eltype(A) T = ArrowType(S) fi = firstindex(x) if S === T && concrete_or_concreteunion(S) && fi == 1 return x elseif !concrete_or_concreteunion(T) # arrow needs concrete types, so try to find a concrete common type, preferring unions if isempty(x) return Missing[] end T = mapreduce(typeof ∘ toarrow, promoteunion, x) if T === Missing && concrete_or_concreteunion(S) T = promoteunion(T, typeof(toarrow(default(S)))) end end return ToArrow{T,A}(x) end Base.IndexStyle(::Type{<:ToArrow}) = Base.IndexLinear() Base.size(x::ToArrow) = (length(x.data),) Base.eltype(::Type{TA}) where {T,A,TA<:ToArrow{T,A}} = T function _convert(::Type{T}, x) where {T} if x isa T return x elseif T isa Union # T was a promoted Union and x is not already one of # the concrete Union types, so we need to just try # to convert, recursively, to one of the Union types # unfortunately not much we can do more efficiently here try return _convert(T.a, x) catch return _convert(T.b, x) end else return convert(T, x) end end Base.getindex(x::ToArrow{T}, i::Int) where {T} = _convert(T, toarrow(getindex(x.data, i + firstindex(x.data) - 1))) end # module ArrowTypes ================================================ FILE: src/ArrowTypes/test/Project.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [deps] OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" ================================================ FILE: src/ArrowTypes/test/runtests.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. using Test, ArrowTypes, UUIDs, Sockets, OffsetArrays include("tests.jl") ================================================ FILE: src/ArrowTypes/test/tests.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. primitive type MyInt 32 end struct Person id::Int name::String end @testset "ArrowTypes" begin @test ArrowTypes.ArrowKind(MyInt) == ArrowTypes.PrimitiveKind() @test ArrowTypes.ArrowKind(Person) == ArrowTypes.StructKind() @test ArrowTypes.ArrowKind(Person(0, "bob")) == ArrowTypes.StructKind() @test ArrowTypes.ArrowType(Int) == Int @test ArrowTypes.ArrowType(Union{Int,Missing}) == Union{Int,Missing} @test ArrowTypes.ArrowType(Missing) == Missing @test ArrowTypes.toarrow(1) === 1 @test ArrowTypes.arrowname(Int) == Symbol() @test !ArrowTypes.hasarrowname(Int) @test ArrowTypes.arrowmetadata(Int) == "" @test ArrowTypes.arrowmetadata(Union{Nothing,Int}) == "" @test ArrowTypes.arrowmetadata(Union{Missing,Int}) == "" @test ArrowTypes.JuliaType(1) === nothing @test ArrowTypes.JuliaType(1, Int) === nothing @test ArrowTypes.JuliaType(1, Int, nothing) === nothing @test ArrowTypes.fromarrow(Int, 1) === 1 @test ArrowTypes.fromarrow(Person, 1, "bob") == Person(1, "bob") @test ArrowTypes.fromarrow(Union{Int,Missing}, missing) === missing @test ArrowTypes.fromarrow(Union{Int,Missing}, 1) === 1 @test ArrowTypes.fromarrow(Union{Float64,Missing}, 1) === 1.0 @test ArrowTypes.ArrowKind(Missing) == ArrowTypes.NullKind() @test ArrowTypes.ArrowKind(Nothing) == ArrowTypes.NullKind() @test ArrowTypes.ArrowType(Nothing) == Missing @test ArrowTypes.toarrow(nothing) === missing @test ArrowTypes.arrowname(Nothing) == ArrowTypes.NOTHING @test ArrowTypes.JuliaType(Val(ArrowTypes.NOTHING)) == Nothing @test ArrowTypes.fromarrow(Nothing, missing) === nothing @test ArrowTypes.ArrowKind(Int) == ArrowTypes.PrimitiveKind() @test ArrowTypes.ArrowKind(Float64) == ArrowTypes.PrimitiveKind() @test ArrowTypes.ArrowType(Char) == UInt32 @test ArrowTypes.toarrow('1') == UInt32('1') @test ArrowTypes.arrowname(Char) == ArrowTypes.CHAR @test ArrowTypes.JuliaType(Val(ArrowTypes.CHAR)) == Char @test ArrowTypes.fromarrow(Char, UInt32('1')) == '1' @test ArrowTypes.ArrowKind(Bool) == ArrowTypes.BoolKind() @test ArrowTypes.ListKind() == ArrowTypes.ListKind{false}() @test !ArrowTypes.isstringtype(ArrowTypes.ListKind()) @test !ArrowTypes.isstringtype(typeof(ArrowTypes.ListKind())) @test ArrowTypes.ArrowKind(String) == ArrowTypes.ListKind{true}() @test ArrowTypes.ArrowKind(Base.CodeUnits) == ArrowTypes.ListKind{true}() hey = collect(b"hey") @test ArrowTypes.fromarrow(String, pointer(hey), 3) == "hey" @test ArrowTypes.fromarrow(Base.CodeUnits, pointer(hey), 3) == b"hey" @test ArrowTypes.fromarrow(Union{Base.CodeUnits,Missing}, pointer(hey), 3) == b"hey" @test ArrowTypes.ArrowType(Symbol) == String @test ArrowTypes.toarrow(:hey) == "hey" @test ArrowTypes.arrowname(Symbol) == ArrowTypes.SYMBOL @test ArrowTypes.JuliaType(Val(ArrowTypes.SYMBOL)) == Symbol @test ArrowTypes.fromarrow(Symbol, pointer(hey), 3) == :hey @test ArrowTypes.ArrowKind(Vector{Int}) == ArrowTypes.ListKind() @test ArrowTypes.ArrowKind(Set{Int}) == ArrowTypes.ListKind() @test ArrowTypes.ArrowType(Set{Int}) == Vector{Int} @test typeof(ArrowTypes.toarrow(Set([1, 2, 3]))) <: Vector{Int} @test ArrowTypes.arrowname(Set{Int}) == ArrowTypes.SET @test ArrowTypes.JuliaType(Val(ArrowTypes.SET), Vector{Int}) == Set{Int} @test ArrowTypes.fromarrow(Set{Int}, [1, 2, 3]) == Set([1, 2, 3]) K = ArrowTypes.ArrowKind(NTuple{3,UInt8}) @test ArrowTypes.gettype(K) == UInt8 @test ArrowTypes.getsize(K) == 3 @test K == ArrowTypes.FixedSizeListKind{3,UInt8}() u = UUID(rand(UInt128)) ubytes = ArrowTypes._cast(NTuple{16,UInt8}, u.value) @test ArrowTypes.ArrowKind(u) == ArrowTypes.FixedSizeListKind{16,UInt8}() @test ArrowTypes.ArrowType(UUID) == NTuple{16,UInt8} @test ArrowTypes.toarrow(u) == ubytes @test ArrowTypes.arrowname(UUID) == ArrowTypes.UUIDSYMBOL @test ArrowTypes.JuliaType(Val(ArrowTypes.UUIDSYMBOL)) == UUID @test ArrowTypes.fromarrow(UUID, ubytes) == u ip4 = IPv4(rand(UInt32)) @test ArrowTypes.ArrowKind(ip4) == PrimitiveKind() @test ArrowTypes.ArrowType(IPv4) == UInt32 @test ArrowTypes.toarrow(ip4) == ip4.host @test ArrowTypes.arrowname(IPv4) == ArrowTypes.IPV4_SYMBOL @test ArrowTypes.JuliaType(Val(ArrowTypes.IPV4_SYMBOL)) == IPv4 @test ArrowTypes.fromarrow(IPv4, ip4.host) == ip4 ip6 = IPv6(rand(UInt128)) ip6_ubytes = ArrowTypes._cast(NTuple{16,UInt8}, ip6.host) @test ArrowTypes.ArrowKind(ip6) == ArrowTypes.FixedSizeListKind{16,UInt8}() @test ArrowTypes.ArrowType(IPv6) == NTuple{16,UInt8} @test ArrowTypes.toarrow(ip6) == ip6_ubytes @test ArrowTypes.arrowname(IPv6) == ArrowTypes.IPV6_SYMBOL @test ArrowTypes.JuliaType(Val(ArrowTypes.IPV6_SYMBOL)) == IPv6 @test ArrowTypes.fromarrow(IPv6, ip6_ubytes) == ip6 nt = (id=1, name="bob") @test ArrowTypes.ArrowKind(NamedTuple) == ArrowTypes.StructKind() @test ArrowTypes.fromarrow(typeof(nt), nt) === nt @test ArrowTypes.fromarrow(Person, nt) == Person(1, "bob") @test ArrowTypes.ArrowKind(Tuple) == ArrowTypes.StructKind() @test ArrowTypes.ArrowKind(Tuple{}) == ArrowTypes.StructKind() @test ArrowTypes.arrowname(Tuple{Int,String}) == ArrowTypes.TUPLE @test ArrowTypes.arrowname(Tuple{}) == ArrowTypes.TUPLE @test ArrowTypes.JuliaType( Val(ArrowTypes.TUPLE), NamedTuple{(Symbol("1"), Symbol("2")),Tuple{Int,String}}, ) == Tuple{Int,String} @test ArrowTypes.fromarrow(Tuple{Int,String}, nt) == (1, "bob") @test ArrowTypes.fromarrow(Union{Missing,typeof(nt)}, nt) == nt # #461 @test ArrowTypes.default(Tuple{}) == () @test ArrowTypes.default(Tuple{Vararg{Int}}) == () @test ArrowTypes.default(Tuple{String,Vararg{Int}}) == ("",) v = v"1" v_nt = (major=1, minor=0, patch=0, prerelease=(), build=()) @test ArrowTypes.ArrowKind(VersionNumber) == ArrowTypes.StructKind() @test ArrowTypes.arrowname(VersionNumber) == ArrowTypes.VERSION_NUMBER @test ArrowTypes.JuliaType(Val(ArrowTypes.VERSION_NUMBER)) == VersionNumber @test ArrowTypes.fromarrow(typeof(v), v_nt) == v @test ArrowTypes.default(VersionNumber) == v"0" @test ArrowTypes.ArrowKind(Dict{String,Int}) == ArrowTypes.MapKind() @test ArrowTypes.ArrowKind(Union{String,Int}) == ArrowTypes.UnionKind() @test ArrowTypes.default(Int) == Int(0) @test ArrowTypes.default(Symbol) == Symbol() @test ArrowTypes.default(Char) == '\0' @test ArrowTypes.default(String) == "" @test ArrowTypes.default(Missing) === missing @test ArrowTypes.default(Nothing) === nothing @test ArrowTypes.default(Union{Int,Missing}) == Int(0) @test ArrowTypes.default(Union{Int,Nothing}) == Int(0) @test ArrowTypes.default(Union{Int,Missing,Nothing}) == Int(0) @test ArrowTypes.promoteunion(Int, Float64) == Float64 @test ArrowTypes.promoteunion(Int, String) == Union{Int,String} @test ArrowTypes.concrete_or_concreteunion(Int) @test !ArrowTypes.concrete_or_concreteunion(Union{Real,String}) @test !ArrowTypes.concrete_or_concreteunion(Any) @testset "ToArrow" begin x = ArrowTypes.ToArrow([1, 2, 3]) @test x isa Vector{Int} @test x == [1, 2, 3] x = ArrowTypes.ToArrow([:hey, :ho]) @test x isa ArrowTypes.ToArrow{String,Vector{Symbol}} @test eltype(x) == String @test x == ["hey", "ho"] x = ArrowTypes.ToArrow(Any[1, 3.14]) @test x isa ArrowTypes.ToArrow{Float64,Vector{Any}} @test eltype(x) == Float64 @test x == [1.0, 3.14] x = ArrowTypes.ToArrow(Any[1, 3.14, "hey"]) @test x isa ArrowTypes.ToArrow{Union{Float64,String},Vector{Any}} @test eltype(x) == Union{Float64,String} @test x == [1.0, 3.14, "hey"] x = ArrowTypes.ToArrow(OffsetArray([1, 2, 3], -3:-1)) @test x isa ArrowTypes.ToArrow{Int,OffsetVector{Int,Vector{Int}}} @test eltype(x) == Int @test x == [1, 2, 3] x = ArrowTypes.ToArrow(OffsetArray(Any[1, 3.14], -3:-2)) @test x isa ArrowTypes.ToArrow{Float64,OffsetVector{Any,Vector{Any}}} @test eltype(x) == Float64 @test x == [1, 3.14] @testset "respect non-missing concrete type" begin struct DateTimeTZ instant::Int64 tz::String end struct Timestamp{TZ} x::Int64 end ArrowTypes.ArrowType(::Type{DateTimeTZ}) = Timestamp ArrowTypes.toarrow(x::DateTimeTZ) = Timestamp{Symbol(x.tz)}(x.instant) ArrowTypes.default(::Type{DateTimeTZ}) = DateTimeTZ(0, "UTC") T = Union{DateTimeTZ,Missing} @test !ArrowTypes.concrete_or_concreteunion(ArrowTypes.ArrowType(T)) @test eltype(ArrowTypes.ToArrow(T[missing])) == Union{Timestamp{:UTC},Missing} # Works since `ArrowTypes.default(Any) === nothing` and # `ArrowTypes.toarrow(nothing) === missing`. Defining `toarrow(::Nothing) = nothing` # would break this test by returning `Union{Nothing,Missing}`. @test eltype(ArrowTypes.ToArrow(Any[missing])) == Missing end @testset "ignore non-missing abstract type" begin x = ArrowTypes.ToArrow(Union{Missing,Array{Int}}[missing]) @test x isa ArrowTypes.ToArrow{Missing,Vector{Union{Missing,Array{Int64}}}} @test eltype(x) == Missing @test isequal(x, [missing]) end end end ================================================ FILE: src/FlatBuffers/FlatBuffers.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. module FlatBuffers import Base: == const UOffsetT = UInt32 const SOffsetT = Int32 const VOffsetT = UInt16 const VtableMetadataFields = 2 basetype(::Enum) = UInt8 basetype(::Type{T}) where {T<:Enum{S}} where {S} = S function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{Bool}) @inbounds b = t[pos + 1] return b === 0x01 end function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{T}) where {T} GC.@preserve t begin ptr = convert(Ptr{T}, pointer(t, pos + 1)) x = unsafe_load(ptr) end end include("builder.jl") include("table.jl") function Base.show(io::IO, x::TableOrStruct) print(io, "$(typeof(x))") if isempty(propertynames(x)) print(io, "()") else show( io, NamedTuple{propertynames(x)}( Tuple(getproperty(x, y) for y in propertynames(x)), ), ) end end end # module ================================================ FILE: src/FlatBuffers/builder.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. const fileIdentifierLength = 4 """ Scalar A Union of the Julia types `T <: Number` that are allowed in FlatBuffers schema """ const Scalar = Union{Bool,Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,Float32,Float64,Enum} """ Builder is a state machine for creating FlatBuffer objects. Use a Builder to construct object(s) starting from leaf nodes. A Builder constructs byte buffers in a last-first manner for simplicity and performance. """ mutable struct Builder bytes::Vector{UInt8} minalign::Int vtable::Vector{UOffsetT} objectend::UOffsetT vtables::Vector{UOffsetT} head::UOffsetT nested::Bool finished::Bool sharedstrings::Dict{String,UOffsetT} end bytes(b::Builder) = getfield(b, :bytes) Builder(size=0) = Builder( zeros(UInt8, size), 1, UOffsetT[], UOffsetT(0), UOffsetT[], UOffsetT(size), false, false, Dict{String,UOffsetT}(), ) function reset!(b::Builder) empty!(b.bytes) empty!(b.vtable) emtpy!(b.vtables) empty!(b.sharedstrings) b.minalign = 1 b.nested = false b.finished = false b.head = 0 return end Base.write(sink::Builder, o, x::Union{Bool,UInt8}) = sink.bytes[o + 1] = UInt8(x) function Base.write(sink::Builder, off, x::T) where {T} off += 1 for (i, ind) in enumerate(off:(off + sizeof(T) - 1)) sink.bytes[ind] = (x >> ((i - 1) * 8)) % UInt8 end end Base.write(b::Builder, o, x::Float32) = write(b, o, reinterpret(UInt32, x)) Base.write(b::Builder, o, x::Float64) = write(b, o, reinterpret(UInt64, x)) Base.write(b::Builder, o, x::Enum) = write(b, o, basetype(x)(x)) """ `finishedbytes` returns a pointer to the written data in the byte buffer. Panics if the builder is not in a finished state (which is caused by calling `finish!()`). """ function finishedbytes(b::Builder) assertfinished(b) return view(b.bytes, (b.head + 1):length(b.bytes)) end function startobject!(b::Builder, numfields) assertnotnested(b) b.nested = true resize!(b.vtable, numfields) fill!(b.vtable, 0) b.objectend = offset(b) return end """ WriteVtable serializes the vtable for the current object, if applicable. Before writing out the vtable, this checks pre-existing vtables for equality to this one. If an equal vtable is found, point the object to the existing vtable and return. Because vtable values are sensitive to alignment of object data, not all logically-equal vtables will be deduplicated. A vtable has the following format: * N, where N is the number of fields in the schema for this type. Includes deprecated fields. Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. An object has the following format: + """ function writevtable!(b::Builder) # Prepend a zero scalar to the object. Later in this function we'll # write an offset here that points to the object's vtable: prepend!(b, SOffsetT(0)) objectOffset = offset(b) existingVtable = UOffsetT(0) # Trim vtable of trailing zeroes. i = findlast(!iszero, b.vtable) if i !== nothing resize!(b.vtable, i) end # Search backwards through existing vtables, because similar vtables # are likely to have been recently appended. See # BenchmarkVtableDeduplication for a case in which this heuristic # saves about 30% of the time used in writing objects with duplicate # tables. for i = length(b.vtables):-1:1 # Find the other vtable, which is associated with `i`: vt2Offset = b.vtables[i] vt2Start = length(b.bytes) - vt2Offset vt2Len = readbuffer(b.bytes, vt2Start, VOffsetT) metadata = VtableMetadataFields * sizeof(VOffsetT) vt2End = vt2Start + vt2Len vt2 = view(b.bytes, (vt2Start + metadata + 1):vt2End) #TODO: might need a +1 on the start of range here # Compare the other vtable to the one under consideration. # If they are equal, store the offset and break: if vtableEqual(b.vtable, objectOffset, vt2) existingVtable = vt2Offset break end end if existingVtable == 0 # Did not find a vtable, so write this one to the buffer. # Write out the current vtable in reverse , because # serialization occurs in last-first order: for i = length(b.vtable):-1:1 off::UOffsetT = 0 if b.vtable[i] != 0 # Forward reference to field; # use 32bit number to assert no overflow: off = objectOffset - b.vtable[i] end prepend!(b, VOffsetT(off)) end # The two metadata fields are written last. # First, store the object bytesize: objectSize = objectOffset - b.objectend prepend!(b, VOffsetT(objectSize)) # Second, store the vtable bytesize: vbytes = (length(b.vtable) + VtableMetadataFields) * sizeof(VOffsetT) prepend!(b, VOffsetT(vbytes)) # Next, write the offset to the new vtable in the # already-allocated SOffsetT at the beginning of this object: objectStart = SOffsetT(length(b.bytes) - objectOffset) write(b, objectStart, SOffsetT(offset(b) - objectOffset)) # Finally, store this vtable in memory for future # deduplication: push!(b.vtables, offset(b)) else # Found a duplicate vtable. objectStart = SOffsetT(length(b.bytes) - objectOffset) b.head = objectStart # Write the offset to the found vtable in the # already-allocated SOffsetT at the beginning of this object: write(b, b.head, SOffsetT(existingVtable) - SOffsetT(objectOffset)) end empty!(b.vtable) return objectOffset end """ `endobject` writes data necessary to finish object construction. """ function endobject!(b::Builder) assertnested(b) n = writevtable!(b) b.nested = false return n end offset(b::Builder) = UOffsetT(length(b.bytes) - b.head) pad!(b::Builder, n) = foreach(x -> place!(b, 0x00), 1:n) """ `prep!` prepares to write an element of `size` after `additionalbytes` have been written, e.g. if you write a string, you need to align such the int length field is aligned to sizeof(Int32), and the string data follows it directly. If all you need to do is align, `additionalbytes` will be 0. """ function prep!(b::Builder, size, additionalbytes) # Track the biggest thing we've ever aligned to. if size > b.minalign b.minalign = size end # Find the amount of alignment needed such that `size` is properly # aligned after `additionalBytes`: alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1 alignsize &= (size - 1) # Reallocate the buffer if needed: totalsize = alignsize + size + additionalbytes if b.head <= totalsize len = length(b.bytes) prepend!(b.bytes, zeros(UInt8, totalsize)) b.head += length(b.bytes) - len end pad!(b, alignsize) return end function Base.prepend!(b::Builder, x::T) where {T} prep!(b, sizeof(T), 0) place!(b, x) return end function prependoffset!(b::Builder, off) prep!(b, sizeof(Int32), 0) # Ensure alignment is already done. if !(off <= offset(b)) throw(ArgumentError("unreachable: $off <= $(offset(b))")) end place!(b, SOffsetT(offset(b) - off + sizeof(SOffsetT))) return end function prependoffsetslot!(b::Builder, o::Int, x::T, d) where {T} if x != T(d) prependoffset!(b, x) slot!(b, o) end return end """ `startvector` initializes bookkeeping for writing a new vector. A vector has the following format: +, where T is the type of elements of this vector. """ function startvector!(b::Builder, elemSize, numElems, alignment) assertnotnested(b) b.nested = true prep!(b, sizeof(UInt32), elemSize * numElems) prep!(b, alignment, elemSize * numElems) return offset(b) end """ `endvector` writes data necessary to finish vector construction. """ function endvector!(b::Builder, vectorNumElems) assertnested(b) place!(b, UOffsetT(vectorNumElems)) b.nested = false return offset(b) end function createsharedstring!(b::Builder, s::AbstractString) get!(b.sharedstrings, s) do createstring!(b, s) end end """ `createstring!` writes a null-terminated string as a vector. """ function createstring!(b::Builder, s::Union{AbstractString,AbstractVector{UInt8}}) assertnotnested(b) b.nested = true s = codeunits(s) prep!(b, sizeof(UInt32), sizeof(s) + 1) place!(b, UInt8(0)) l = sizeof(s) b.head -= l copyto!(b.bytes, b.head + 1, s, 1, l) return endvector!(b, sizeof(s)) end createbytevector(b::Builder, v) = createstring!(b, v) function assertnested(b::Builder) # If you get this assert, you're in an object while trying to write # data that belongs outside of an object. # To fix this, write non-inline data (like vectors) before creating # objects. if !b.nested throw(ArgumentError("Incorrect creation order: must be inside object.")) end return end function assertnotnested(b::Builder) # If you hit this, you're trying to construct a Table/Vector/String # during the construction of its parent table (between the MyTableBuilder # and builder.Finish()). # Move the creation of these view-objects to above the MyTableBuilder to # not get this assert. # Ignoring this assert may appear to work in simple cases, but the reason # it is here is that storing objects in-line may cause vtable offsets # to not fit anymore. It also leads to vtable duplication. if b.nested throw(ArgumentError("Incorrect creation order: object must not be nested.")) end return end function assertfinished(b::Builder) # If you get this assert, you're attempting to get access a buffer # which hasn't been finished yet. Be sure to call builder.Finish() # with your root table. # If you really need to access an unfinished buffer, use the bytes # buffer directly. if !b.finished throw(ArgumentError("Incorrect use of FinishedBytes(): must call 'Finish' first.")) end end """ `prependslot!` prepends a `T` onto the object at vtable slot `o`. If value `x` equals default `d`, then the slot will be set to zero and no other data will be written. """ function prependslot!(b::Builder, o::Int, x::T, d, sh=false) where {T<:Scalar} if x != T(d) prepend!(b, x) slot!(b, o) end return end """ `prependstructslot!` prepends a struct onto the object at vtable slot `o`. Structs are stored inline, so nothing additional is being added. In generated code, `d` is always 0. """ function prependstructslot!(b::Builder, voffset, x, d) if x != d assertnested(b) if x != offset(b) throw(ArgumentError("inline data write outside of object")) end slot!(b, voffset) end return end """ `slot!` sets the vtable key `voffset` to the current location in the buffer. """ function slot!(b::Builder, slotnum) b.vtable[slotnum + 1] = offset(b) end # FinishWithFileIdentifier finalizes a buffer, pointing to the given `rootTable`. # as well as applys a file identifier function finishwithfileidentifier(b::Builder, rootTable, fid) if length(fid) != fileIdentifierLength error("incorrect file identifier length") end # In order to add a file identifier to the flatbuffer message, we need # to prepare an alignment and file identifier length prep!(b, b.minalign, sizeof(Int32) + fileIdentifierLength) for i = fileIdentifierLength:-1:1 # place the file identifier place!(b, fid[i]) end # finish finish!(b, rootTable) end """ `finish!` finalizes a buffer, pointing to the given `rootTable`. """ function finish!(b::Builder, rootTable) assertnotnested(b) prep!(b, b.minalign, sizeof(UOffsetT)) prependoffset!(b, UOffsetT(rootTable)) b.finished = true return end "vtableEqual compares an unwritten vtable to a written vtable." function vtableEqual(a::Vector{UOffsetT}, objectStart, b::AbstractVector{UInt8}) if length(a) * sizeof(VOffsetT) != length(b) return false end for i = 0:(length(a) - 1) x = read(IOBuffer(view(b, (i * sizeof(VOffsetT) + 1):length(b))), VOffsetT) # Skip vtable entries that indicate a default value. x == 0 && a[i + 1] == 0 && continue y = objectStart - a[i + 1] x != y && return false end return true end """ `place!` prepends a `T` to the Builder, without checking for space. """ function place!(b::Builder, x::T) where {T} b.head -= sizeof(T) write(b, b.head, x) return end ================================================ FILE: src/FlatBuffers/table.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Table The object containing the flatbuffer and positional information specific to the table. The `vtable` containing the offsets for specific members precedes `pos`. The actual values in the table follow `pos` offset and size of the vtable. - `bytes::Vector{UInt8}`: the flatbuffer itself - `pos::Integer`: the base position in `bytes` of the table """ abstract type Table end abstract type Struct end const TableOrStruct = Union{Table,Struct} bytes(x::TableOrStruct) = getfield(x, :bytes) pos(x::TableOrStruct) = getfield(x, :pos) ==(a::T, b::T) where {T<:TableOrStruct} = all(getproperty(a, p) == getproperty(b, p) for p in propertynames(a)) (::Type{T})(b::Builder) where {T<:TableOrStruct} = T(b.bytes[(b.head + 1):end], get(b, b.head, Int32)) getrootas(::Type{T}, bytes::Vector{UInt8}, offset) where {T<:Table} = init(T, bytes, offset + readbuffer(bytes, offset, UOffsetT)) init(::Type{T}, bytes::Vector{UInt8}, pos::Integer) where {T<:TableOrStruct} = T(bytes, pos) const TableOrBuilder = Union{Table,Struct,Builder} Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T} = readbuffer(bytes(t), pos, T) Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T<:Enum} = T(get(t, pos, basetype(T))) """ `offset` provides access into the Table's vtable. Deprecated fields are ignored by checking against the vtable's length. """ function offset(t::Table, vtableoffset) vtable = pos(t) - get(t, pos(t), SOffsetT) return vtableoffset < get(t, vtable, VOffsetT) ? get(t, vtable + vtableoffset, VOffsetT) : VOffsetT(0) end "`indirect` retrieves the relative offset stored at `offset`." indirect(t::Table, off) = off + get(t, off, UOffsetT) getvalue(t, o, ::Type{Nothing}) = nothing getvalue(t, o, ::Type{T}) where {T<:Scalar} = get(t, pos(t) + o, T) getvalue(t, o, ::Type{T}) where {T<:Enum} = T(get(t, pos(t) + o, enumtype(T))) function Base.String(t::Table, off) off += get(t, off, UOffsetT) start = off + sizeof(UOffsetT) len = get(t, off, UOffsetT) return unsafe_string(pointer(bytes(t), start + 1), len) end function bytevector(t::Table, off) off += get(t, off, UOffsetT) start = off + sizeof(UOffsetT) len = get(t, off, UOffsetT) return view(bytes(t), (start + 1):(start + len + 1)) end """ `vectorlen` retrieves the length of the vector whose offset is stored at `off` in this object. """ function vectorlen(t::Table, off) off += pos(t) off += get(t, off, UOffsetT) return Int(get(t, off, UOffsetT)) end """ `vector` retrieves the start of data of the vector whose offset is stored at `off` in this object. """ function vector(t::Table, off) off += pos(t) x = off + get(t, off, UOffsetT) # data starts after metadata containing the vector length return x + sizeof(UOffsetT) end struct Array{T,S,TT} <: AbstractVector{T} _tab::TT pos::Int64 data::Vector{S} end function Array{T}(t::Table, off) where {T} a = vector(t, off) S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T),UInt8} : T ptr = convert(Ptr{S}, pointer(bytes(t), a + 1)) data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off)) return Array{T,S,typeof(t)}(t, a, data) end function structsizeof end Base.IndexStyle(::Type{<:Array}) = Base.IndexLinear() Base.size(x::Array) = size(x.data) Base.@propagate_inbounds function Base.getindex(A::Array{T,S}, i::Integer) where {T,S} if T === S return A.data[i] elseif T <: Struct return init(T, bytes(A._tab), A.pos + (i - 1) * structsizeof(T)) else # T isa Table return init(T, bytes(A._tab), indirect(A._tab, A.pos + (i - 1) * 4)) end end Base.@propagate_inbounds function Base.setindex!(A::Array{T,S}, v, i::Integer) where {T,S} if T === S return setindex!(A.data, v, i) else error("setindex! not supported for reference/table types") end end function union(t::Table, off) off += pos(t) return off + get(t, off, UOffsetT) end function union!(t::Table, t2::Table, off) off += pos(t) t2.pos = off + get(t, off, UOffsetT) t2.bytes = bytes(t) return end """ GetVOffsetTSlot retrieves the VOffsetT that the given vtable location points to. If the vtable value is zero, the default value `d` will be returned. """ function getoffsetslot(t::Table, slot, d) off = offset(t, slot) if off == 0 return d end return off end """ `getslot` retrieves the `T` that the given vtable location points to. If the vtable value is zero, the default value `d` will be returned. """ function getslot(t::Table, slot, d::T) where {T} off = offset(t, slot) if off == 0 return d end return get(t, pos(t) + off, T) end ================================================ FILE: src/append.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Arrow.append(io::IO, tbl) Arrow.append(file::String, tbl) tbl |> Arrow.append(file) Append any [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible `tbl` to an existing arrow formatted file or IO. The existing arrow data must be in IPC stream format. Note that appending to the "feather formatted file" is _not_ allowed, as this file format doesn't support appending. That means files written like `Arrow.write(filename::String, tbl)` _cannot_ be appended to; instead, you should write like `Arrow.write(filename::String, tbl; file=false)`. When an IO object is provided to be written on to, it must support seeking. For example, a file opened in `r+` mode or an `IOBuffer` that is readable, writable and seekable can be appended to, but not a network stream. Multiple record batches will be written based on the number of `Tables.partitions(tbl)` that are provided; by default, this is just one for a given table, but some table sources support automatic partitioning. Note you can turn multiple table objects into partitions by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that each table must have the exact same `Tables.Schema`. By default, `Arrow.append` will use multiple threads to write multiple record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set). Supported keyword arguments to `Arrow.append` include: * `alignment::Int=8`: specify the number of bytes to align buffers to when written in messages; strongly recommended to only use alignment values of 8 or 64 for modern memory cache line optimization * `colmetadata=nothing`: the metadata that should be written as the table's columns' `custom_metadata` fields; must either be `nothing` or an `AbstractDict` of `column_name::Symbol => column_metadata` where `column_metadata` is an iterable of `<:AbstractString` pairs. * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written; to dict encode specific columns, wrap the column/array in `Arrow.DictEncode(col)` * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; other language implementations [may not support this](https://arrow.apache.org/docs/status.html) * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed * `maxdepth::Int=$DEFAULT_MAX_DEPTH`: deepest allowed nested serialization level; this is provided by default to prevent accidental infinite recursion with mutually recursive data structures * `metadata=Arrow.getmetadata(tbl)`: the metadata that should be written as the table's schema's `custom_metadata` field; must either be `nothing` or an iterable of `<:AbstractString` pairs. * `ntasks::Int`: number of concurrent threaded tasks to allow while writing input partitions out as arrow record batches; default is no limit; to disable multithreaded writing, pass `ntasks=1` * `convert::Bool`: whether certain arrow primitive types in the schema of `file` should be converted to Julia defaults for matching them to the schema of `tbl`; by default, `convert=true`. * `file::Bool`: applicable when an `IO` is provided, whether it is a file; by default `file=false`. """ function append end append(io_or_file; kw...) = x -> append(io_or_file, x; kw...) function append(file::String, tbl; kwargs...) open(file, isfile(file) ? "r+" : "w+") do io append(io, tbl; file=true, kwargs...) end return file end function append( io::IO, tbl; metadata=getmetadata(tbl), colmetadata=nothing, largelists::Bool=false, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Int=8, maxdepth::Int=DEFAULT_MAX_DEPTH, ntasks=Inf, convert::Bool=true, file::Bool=false, ) if ntasks < 1 throw( ArgumentError( "ntasks keyword argument must be > 0; pass `ntasks=1` to disable multithreaded writing", ), ) end startpos = position(io) seekend(io) len = position(io) - startpos seek(io, startpos) # leave the stream position unchanged if len == 0 # empty file, not initialized, we can just write to it kwargs = Dict{Symbol,Any}( :largelists => largelists, :denseunions => denseunions, :dictencode => dictencode, :dictencodenested => dictencodenested, :alignment => alignment, :maxdepth => maxdepth, :metadata => metadata, :colmetadata => colmetadata, ) if isa(ntasks, Integer) kwargs[:ntasks] = ntasks end write(io, tbl; kwargs...) else isstream, arrow_schema, compress = stream_properties(io; convert=convert) if !isstream throw(ArgumentError("append is supported only to files in arrow stream format")) end if compress isa Symbol && compress !== :lz4 && compress !== :zstd throw( ArgumentError( "unsupported compress keyword argument value: $compress. Valid values include `:lz4` or `:zstd`", ), ) end append( io, tbl, arrow_schema, compress, largelists, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, metadata, colmetadata, ) end return io end function append( io::IO, source, arrow_schema, compress, largelists, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, meta, colmeta, ) seekend(io) skip(io, -8) # overwrite last 8 bytes of last empty message footer sch = Ref{Tables.Schema}(arrow_schema) sync = OrderedSynchronizer() msgs = Channel{Message}(ntasks) dictencodings = Dict{Int64,Any}() # Lockable{DictEncoding} # build messages blocks = (Block[], Block[]) # start message writing from channel threaded = ntasks > 1 tsk = threaded ? (@wkspawn for msg in msgs Base.write(io, msg, blocks, sch, alignment) end) : (@async for msg in msgs Base.write(io, msg, blocks, sch, alignment) end) anyerror = Threads.Atomic{Bool}(false) errorref = Ref{Any}() @sync for (i, tbl) in enumerate(Tables.partitions(source)) if anyerror[] @error "error writing arrow data on partition = $(errorref[][3])" exception = (errorref[][1], errorref[][2]) error("fatal error writing arrow data") end @debug "processing table partition i = $i" tbl_cols = Tables.columns(tbl) tbl_schema = Tables.schema(tbl_cols) if !is_equivalent_schema(arrow_schema, tbl_schema) throw(ArgumentError("Table schema does not match existing arrow file schema")) end if threaded @wkspawn process_partition( tbl_cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, sync, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta, ) else @async process_partition( tbl_cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, sync, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta, ) end end if anyerror[] @error "error writing arrow data on partition = $(errorref[][3])" exception = (errorref[][1], errorref[][2]) error("fatal error writing arrow data") end # close our message-writing channel, no further put!-ing is allowed close(msgs) # now wait for our message-writing task to finish writing wait(tsk) Base.write( io, Message(UInt8[], nothing, 0, true, false, Meta.Schema), blocks, sch, alignment, ) return io end function stream_properties(io::IO; convert::Bool=true) startpos = position(io) buff = similar(FILE_FORMAT_MAGIC_BYTES) start_magic = read!(io, buff) == FILE_FORMAT_MAGIC_BYTES seekend(io) len = position(io) - startpos skip(io, -length(FILE_FORMAT_MAGIC_BYTES)) end_magic = read!(io, buff) == FILE_FORMAT_MAGIC_BYTES seek(io, startpos) # leave the stream position unchanged isstream = !(len > 24 && start_magic && end_magic) if isstream stream = Stream(io, convert=convert) for table in stream # no need to scan further once we get compression information (stream.compression[] !== nothing) && break end seek(io, startpos) # leave the stream position unchanged return isstream, Tables.Schema(stream.names, stream.types), stream.compression[] else return isstream, nothing, nothing end end function is_equivalent_schema(sch1::Tables.Schema, sch2::Tables.Schema) (sch1.names == sch2.names) || (return false) for (t1, t2) in zip(sch1.types, sch2.types) tt1 = Base.nonmissingtype(t1) tt2 = Base.nonmissingtype(t2) if t1 == t2 continue elseif tt1 <: AbstractVector && tt2 <: AbstractVector && eltype(tt1) == eltype(tt2) continue elseif isstructtype(tt1) && isstructtype(tt2) is_equivalent_type_by_field(tt1, tt2) else return false end end true end function is_equivalent_type_by_field(T1, T2) n1 = fieldcount(T1) n2 = fieldcount(T2) n1 != n2 && return false for i = 1:n1 fieldname(T1, i) == fieldname(T2, i) || return false if fieldtype(T1, i) == fieldtype(T2, i) continue elseif isstructtype(T1) && isstructtype(T2) is_equivalent_type_by_field(T1, T2) || continue else return false end end true end ================================================ FILE: src/arraytypes/arraytypes.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.ArrowVector An abstract type that subtypes `AbstractVector`. Each specific arrow array type subtypes `ArrowVector`. See [`BoolVector`](@ref), [`Primitive`](@ref), [`List`](@ref), [`Map`](@ref), [`FixedSizeList`](@ref), [`Struct`](@ref), [`DenseUnion`](@ref), [`SparseUnion`](@ref), and [`DictEncoded`](@ref) for more details. """ abstract type ArrowVector{T} <: AbstractVector{T} end Base.IndexStyle(::Type{A}) where {A<:ArrowVector} = Base.IndexLinear() Base.similar(::Type{A}, dims::Dims) where {T,A<:ArrowVector{T}} = Vector{T}(undef, dims) validitybitmap(x::ArrowVector) = x.validity nullcount(x::ArrowVector) = validitybitmap(x).nc getmetadata(x::ArrowVector) = x.metadata Base.deleteat!(x::T, inds) where {T<:ArrowVector} = throw( ArgumentError("`$T` does not support `deleteat!`; arrow data is by nature immutable"), ) function toarrowvector( x, i=1, de=Dict{Int64,Any}(), ded=DictEncoding[], meta=getmetadata(x); compression::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor}=nothing, kw..., ) @debug "converting top-level column to arrow format: col = $(typeof(x)), compression = $compression, kw = $(values(kw))" @debug x A = arrowvector(x, i, 0, 0, de, ded, meta; compression=compression, kw...) if compression isa LZ4FrameCompressor A = compress(Meta.CompressionType.LZ4_FRAME, compression, A) elseif compression isa ZstdCompressor A = compress(Meta.CompressionType.ZSTD, compression, A) elseif compression isa Symbol && compression == :lz4 comp = lz4_frame_compressor() A = Base.@lock comp begin compress(Meta.CompressionType.LZ4_FRAME, comp[], A) end elseif compression isa Symbol && compression == :zstd comp = zstd_compressor() A = Base.@lock comp begin compress(Meta.CompressionType.ZSTD, comp[], A) end end @debug "converted top-level column to arrow format: $(typeof(A))" @debug A return A end function arrowvector( x, i, nl, fi, de, ded, meta; dictencoding::Bool=false, dictencode::Bool=false, maxdepth::Int=DEFAULT_MAX_DEPTH, kw..., ) if nl > maxdepth error( "reached nested serialization level ($nl) deeper than provided max depth argument ($(maxdepth)); to increase allowed nesting level, pass `maxdepth=X`", ) end T = maybemissing(eltype(x)) if !(x isa DictEncode) && !dictencoding && (dictencode || DataAPI.refarray(x) !== x) x = DictEncode(x, dictencodeid(i, nl, fi)) elseif x isa DictEncoded return arrowvector( DictEncodeType, x, i, nl, fi, de, ded, meta; dictencode=dictencode, kw..., ) elseif !(x isa DictEncode) x = ToArrow(x) end S = maybemissing(eltype(x)) if ArrowTypes.hasarrowname(T) meta = _arrowtypemeta( _normalizemeta(meta), String(ArrowTypes.arrowname(T)), String(ArrowTypes.arrowmetadata(T)), ) end return arrowvector( S, x, i, nl, fi, de, ded, meta; dictencode=dictencode, maxdepth=maxdepth, kw..., ) end _normalizemeta(::Nothing) = nothing _normalizemeta(meta) = toidict(String(k) => String(v) for (k, v) in meta) _normalizecolmeta(::Nothing) = nothing _normalizecolmeta(colmeta) = toidict( Symbol(k) => toidict(String(v1) => String(v2) for (v1, v2) in v) for (k, v) in colmeta ) function _arrowtypemeta(::Nothing, n, m) return toidict(("ARROW:extension:name" => n, "ARROW:extension:metadata" => m)) end function _arrowtypemeta(meta, n, m) dict = Dict(meta) dict["ARROW:extension:name"] = n dict["ARROW:extension:metadata"] = m return toidict(dict) end # now we check for ArrowType converions and dispatch on ArrowKind function arrowvector(::Type{S}, x, i, nl, fi, de, ded, meta; kw...) where {S} meta = _normalizemeta(meta) return arrowvector(ArrowKind(S), x, i, nl, fi, de, ded, meta; kw...) end struct NullVector{T} <: ArrowVector{T} data::MissingVector metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(v::NullVector) = (length(v.data),) Base.getindex(v::NullVector{T}, i::Int) where {T} = ArrowTypes.fromarrow(T, getindex(v.data, i)) arrowvector(::NullKind, x, i, nl, fi, de, ded, meta; kw...) = NullVector{eltype(x)}( MissingVector(length(x)), isnothing(meta) ? nothing : toidict(meta), ) compress(Z::Meta.CompressionType.T, comp, v::NullVector) = Compressed{Z,NullVector}(v, CompressedBuffer[], length(v), length(v), Compressed[]) function makenodesbuffers!( col::NullVector, fieldnodes, fieldbuffers, bufferoffset, alignment, ) push!(fieldnodes, FieldNode(length(col), length(col))) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" return bufferoffset end function writebuffer(io, col::NullVector, alignment) return end """ Arrow.ValidityBitmap A bit-packed array type where each bit corresponds to an element in an [`ArrowVector`](@ref), indicating whether that element is "valid" (bit == 1), or not (bit == 0). Used to indicate element missingness (whether it's null). If the null count of an array is zero, the `ValidityBitmap` will be "empty" and all elements are treated as "valid"/non-null. """ struct ValidityBitmap <: ArrowVector{Bool} bytes::Vector{UInt8} # arrow memory blob pos::Int # starting byte of validity bitmap ℓ::Int # # of _elements_ (not bytes!) in bitmap (because bitpacking) nc::Int # null count end Base.size(p::ValidityBitmap) = (p.ℓ,) nullcount(x::ValidityBitmap) = x.nc function ValidityBitmap(x) T = eltype(x) if !(T >: Missing) return ValidityBitmap(UInt8[], 1, length(x), 0) end len = length(x) blen = cld(len, 8) bytes = Vector{UInt8}(undef, blen) st = iterate(x) nc = 0 b = 0xff j = k = 1 for y in x if y === missing nc += 1 b = setbit(b, false, j) end j += 1 if j == 9 @inbounds bytes[k] = b b = 0xff j = 1 k += 1 end end if j > 1 bytes[k] = b end return ValidityBitmap(nc == 0 ? UInt8[] : bytes, 1, nc == 0 ? 0 : len, nc) end @propagate_inbounds function Base.getindex(p::ValidityBitmap, i::Integer) # no boundscheck because parent array should do it # if a validity bitmap is empty, it either means: # 1) the parent array null_count is 0, so all elements are valid # 2) parent array is also empty, so "all" elements are valid p.nc == 0 && return true # translate element index to bitpacked byte index a, b = divrem(i - 1, 8) .+ (1, 1) @inbounds byte = p.bytes[p.pos + a - 1] # check individual bit of byte return getbit(byte, b) end @propagate_inbounds function Base.setindex!(p::ValidityBitmap, v, i::Integer) x = convert(Bool, v) p.ℓ == 0 && !x && throw(BoundsError(p, i)) a, b = fldmod1(i, 8) @inbounds byte = p.bytes[p.pos + a - 1] @inbounds p.bytes[p.pos + a - 1] = setbit(byte, x, b) return v end function writebitmap(io, col::ArrowVector, alignment) v = col.validity @debug "writing validity bitmap: nc = $(v.nc), n = $(cld(v.ℓ, 8))" v.nc == 0 && return 0 n = Base.write(io, view(v.bytes, (v.pos):(v.pos + cld(v.ℓ, 8) - 1))) return n + writezeros(io, paddinglength(n, alignment)) end include("compressed.jl") include("primitive.jl") include("bool.jl") include("list.jl") include("fixedsizelist.jl") include("map.jl") include("struct.jl") include("unions.jl") include("dictencoding.jl") include("views.jl") ================================================ FILE: src/arraytypes/bool.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.BoolVector A bit-packed array type, similar to [`ValidityBitmap`](@ref), but which holds boolean values, `true` or `false`. """ struct BoolVector{T} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob pos::Int validity::ValidityBitmap ℓ::Int64 metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(p::BoolVector) = (p.ℓ,) @propagate_inbounds function Base.getindex(p::BoolVector{T}, i::Integer) where {T} @boundscheck checkbounds(p, i) if T >: Missing @inbounds !p.validity[i] && return missing end a, b = fldmod1(i, 8) @inbounds byte = p.arrow[p.pos + a - 1] # check individual bit of byte return ArrowTypes.fromarrow(T, getbit(byte, b)) end @propagate_inbounds function Base.setindex!(p::BoolVector, v, i::Integer) @boundscheck checkbounds(p, i) x = convert(Bool, v) a, b = fldmod1(i, 8) @inbounds byte = p.arrow[p.pos + a - 1] @inbounds p.arrow[p.pos + a - 1] = setbit(byte, x, b) return v end arrowvector(::BoolKind, x::BoolVector, i, nl, fi, de, ded, meta; kw...) = x function arrowvector(::BoolKind, x, i, nl, fi, de, ded, meta; kw...) validity = ValidityBitmap(x) len = length(x) blen = cld(len, 8) bytes = Vector{UInt8}(undef, blen) b = 0xff j = k = 1 for y in x if y === false b = setbit(b, false, j) end j += 1 if j == 9 @inbounds bytes[k] = b b = 0xff j = 1 k += 1 end end if j > 1 bytes[k] = b end return BoolVector{eltype(x)}(bytes, 1, validity, len, meta) end function compress(Z::Meta.CompressionType.T, comp, p::P) where {P<:BoolVector} len = length(p) nc = nullcount(p) validity = compress(Z, comp, p.validity) data = compress(Z, comp, view(p.arrow, (p.pos):(p.pos + cld(p.ℓ, 8) - 1))) return Compressed{Z,P}(p, [validity, data], len, nc, Compressed[]) end function makenodesbuffers!( col::BoolVector, fieldnodes, fieldbuffers, bufferoffset, alignment, ) len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" # adjust buffer offset, make primitive array buffer bufferoffset += blen blen = bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" return bufferoffset + blen end function writebuffer(io, col::BoolVector, alignment) @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) n = Base.write(io, view(col.arrow, (col.pos):(col.pos + cld(col.ℓ, 8) - 1))) return n + writezeros(io, paddinglength(n, alignment)) end ================================================ FILE: src/arraytypes/compressed.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct CompressedBuffer data::Vector{UInt8} uncompressedlength::Int64 end """ Arrow.Compressed Represents the compressed version of an [`ArrowVector`](@ref). Holds a reference to the original column. May have `Compressed` children for nested array types. """ struct Compressed{Z,A} data::A buffers::Vector{CompressedBuffer} len::Int64 nullcount::Int64 children::Vector{Compressed} end Base.length(c::Compressed) = c.len Base.eltype(::Type{C}) where {Z,A,C<:Compressed{Z,A}} = eltype(A) getmetadata(x::Compressed) = getmetadata(x.data) compressiontype(c::Compressed{Z}) where {Z} = Z function compress(Z::Meta.CompressionType.T, comp, x::Array) GC.@preserve x begin y = unsafe_wrap(Array, convert(Ptr{UInt8}, pointer(x)), sizeof(x)) return CompressedBuffer(transcode(comp, y), length(y)) end end compress(Z::Meta.CompressionType.T, comp, x) = compress(Z, comp, convert(Array, x)) compress(Z::Meta.CompressionType.T, comp, v::ValidityBitmap) = v.nc == 0 ? CompressedBuffer(UInt8[], 0) : compress(Z, comp, view(v.bytes, (v.pos):(v.pos + cld(v.ℓ, 8) - 1))) function makenodesbuffers!( col::Compressed, fieldnodes, fieldbuffers, bufferoffset, alignment, ) push!(fieldnodes, FieldNode(col.len, col.nullcount)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" for buffer in col.buffers blen = length(buffer.data) == 0 ? 0 : 8 + length(buffer.data) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) end for child in col.children bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment) end return bufferoffset end function writearray(io, b::CompressedBuffer) if length(b.data) > 0 n = Base.write(io, b.uncompressedlength) @debug "writing compressed buffer: uncompressedlength = $(b.uncompressedlength), n = $(length(b.data))" @debug b.data return n + Base.write(io, b.data) end return 0 end function writebuffer(io, col::Compressed, alignment) @debug "writebuffer: col = $(typeof(col))" @debug col for buffer in col.buffers n = writearray(io, buffer) writezeros(io, paddinglength(n, alignment)) end for child in col.children writebuffer(io, child, alignment) end return end ================================================ FILE: src/arraytypes/dictencoding.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.DictEncoding Represents the "pool" of possible values for a [`DictEncoded`](@ref) array type. Whether the order of values is significant can be checked by looking at the `isOrdered` boolean field. The `S` type parameter, while not tied directly to any field, is the signed integer "index type" of the parent DictEncoded. We keep track of this in the DictEncoding in order to validate the length of the pool doesn't exceed the index type limit. The general workflow of writing arrow data means the initial schema will typically be based off the data in the first record batch, and subsequent record batches need to match the same schema exactly. For example, if a non-first record batch dict encoded column were to cause a DictEncoding pool to overflow on unique values, a fatal error should be thrown. """ mutable struct DictEncoding{T,S,A} <: ArrowVector{T} id::Int64 data::A isOrdered::Bool metadata::Union{Nothing,Base.ImmutableDict{String,String}} end indextype(::Type{DictEncoding{T,S,A}}) where {T,S,A} = S indextype(::T) where {T<:DictEncoding} = indextype(T) Base.size(d::DictEncoding) = size(d.data) @propagate_inbounds function Base.getindex(d::DictEncoding{T}, i::Integer) where {T} @boundscheck checkbounds(d, i) return @inbounds ArrowTypes.fromarrow(T, d.data[i]) end # convenience wrapper to signal that an input column should be # dict encoded when written to the arrow format struct DictEncodeType{T} end getT(::Type{DictEncodeType{T}}) where {T} = T """ Arrow.DictEncode(::AbstractVector, id::Integer=nothing) Signals that a column/array should be dictionary encoded when serialized to the arrow streaming/file format. An optional `id` number may be provided to signal that multiple columns should use the same pool when being dictionary encoded. """ struct DictEncode{T,A} <: AbstractVector{DictEncodeType{T}} id::Int64 data::A end DictEncode(x::A, id=-1) where {A} = DictEncode{eltype(A),A}(id, x) Base.IndexStyle(::Type{<:DictEncode}) = Base.IndexLinear() Base.size(x::DictEncode) = (length(x.data),) Base.iterate(x::DictEncode, st...) = iterate(x.data, st...) Base.getindex(x::DictEncode, i::Int) = getindex(x.data, i) ArrowTypes.ArrowKind(::Type{<:DictEncodeType}) = DictEncodedKind() Base.copy(x::DictEncode) = DictEncode(x.data, x.id) """ Arrow.DictEncoded A dictionary encoded array type (similar to a `PooledArray`). Behaves just like a normal array in most respects; internally, possible values are stored in the `encoding::DictEncoding` field, while the `indices::Vector{<:Integer}` field holds the "codes" of each element for indexing into the encoding pool. Any column/array can be dict encoding when serializing to the arrow format either by passing the `dictencode=true` keyword argument to [`Arrow.write`](@ref) (which causes _all_ columns to be dict encoded), or wrapping individual columns/ arrays in [`Arrow.DictEncode(x)`](@ref). """ struct DictEncoded{T,S,A} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob validity::ValidityBitmap indices::Vector{S} encoding::DictEncoding{T,S,A} metadata::Union{Nothing,Base.ImmutableDict{String,String}} end DictEncoded( b::Vector{UInt8}, v::ValidityBitmap, inds::Vector{S}, encoding::DictEncoding{T,S,A}, meta, ) where {S,T,A} = DictEncoded{T,S,A}(b, v, inds, encoding, meta) Base.size(d::DictEncoded) = size(d.indices) isdictencoded(d::DictEncoded) = true isdictencoded(x) = false isdictencoded(c::Compressed{Z,A}) where {Z,A<:DictEncoded} = true function signedtype(n::Integer) typs = (Int8, Int16, Int32, Int64) typs[something(findfirst(n .≤ typemax.(typs)), 4)] end signedtype(::Type{UInt8}) = Int8 signedtype(::Type{UInt16}) = Int16 signedtype(::Type{UInt32}) = Int32 signedtype(::Type{UInt64}) = Int64 signedtype(::Type{T}) where {T<:Signed} = T indtype(d::DictEncoded{T,S,A}) where {T,S,A} = S indtype(c::Compressed{Z,A}) where {Z,A<:DictEncoded} = indtype(c.data) dictencodeid(colidx, nestedlevel, fieldid) = (Int64(nestedlevel) << 48) | (Int64(fieldid) << 32) | Int64(colidx) getid(d::DictEncoded) = d.encoding.id getid(c::Compressed{Z,A}) where {Z,A<:DictEncoded} = c.data.encoding.id function arrowvector( ::DictEncodedKind, x::DictEncoded, i, nl, fi, de, ded, meta; dictencode::Bool=false, dictencodenested::Bool=false, kw..., ) id = x.encoding.id # XXX This is a race condition if two workers hit this block at the same time, then they'll create # distinct locks if !haskey(de, id) de[id] = Lockable(x.encoding) else encodinglockable = de[id] Base.@lock encodinglockable begin encoding = encodinglockable.value # in this case, we just need to check if any values in our local pool need to be delta dicationary serialized deltas = setdiff(x.encoding, encoding) if !isempty(deltas) ET = indextype(encoding) if length(deltas) + length(encoding) > typemax(ET) error( "fatal error serializing dict encoded column with ref index type of $ET; subsequent record batch unique values resulted in $(length(deltas) + length(encoding)) unique values, which exceeds possible index values in $ET", ) end data = arrowvector( deltas, i, nl, fi, de, ded, nothing; dictencode=dictencodenested, dictencodenested=dictencodenested, dictencoding=true, kw..., ) push!( ded, DictEncoding{eltype(data),ET,typeof(data)}( id, data, false, getmetadata(data), ), ) if typeof(encoding.data) <: ChainedVector append!(encoding.data, data) else data2 = ChainedVector([encoding.data, data]) encoding = DictEncoding{eltype(data2),ET,typeof(data2)}( id, data2, false, getmetadata(encoding), ) de[id] = Lockable(encoding) end end end end return x end function arrowvector( ::DictEncodedKind, x, i, nl, fi, de, ded, meta; dictencode::Bool=false, dictencodenested::Bool=false, kw..., ) @assert x isa DictEncode id = x.id == -1 ? dictencodeid(i, nl, fi) : x.id x = x.data len = length(x) validity = ValidityBitmap(x) # XXX This is a race condition if two workers hit this block at the same time, then they'll create # distinct locks if !haskey(de, id) # dict encoding doesn't exist yet, so create for 1st time if DataAPI.refarray(x) === x || DataAPI.refpool(x) === nothing # need to encode ourselves x = PooledArray(x; signed=true, compress=true) inds = refa = DataAPI.refarray(x) pool = DataAPI.refpool(x) else pool = DataAPI.refpool(x) refa = DataAPI.refarray(x) inds = copyto!(similar(Vector{signedtype(length(pool))}, length(refa)), refa) end # adjust to "offset" instead of index inds .-= firstindex(refa) data = arrowvector( pool, i, nl, fi, de, ded, nothing; dictencode=dictencodenested, dictencodenested=dictencodenested, dictencoding=true, kw..., ) encoding = DictEncoding{eltype(data),eltype(inds),typeof(data)}( id, data, false, getmetadata(data), ) de[id] = Lockable(encoding) else # encoding already exists # compute inds based on it # if value doesn't exist in encoding, push! it # also add to deltas updates encodinglockable = de[id] Base.@lock encodinglockable begin encoding = encodinglockable.value len = length(x) ET = indextype(encoding) pool = Dict{Union{eltype(encoding),eltype(x)},ET}( a => (b - 1) for (b, a) in enumerate(encoding) ) deltas = eltype(x)[] inds = Vector{ET}(undef, len) for (j, val) in enumerate(x) @inbounds inds[j] = get!(pool, val) do push!(deltas, val) return length(pool) end end if !isempty(deltas) if length(deltas) + length(encoding) > typemax(ET) error( "fatal error serializing dict encoded column with ref index type of $ET; subsequent record batch unique values resulted in $(length(deltas) + length(encoding)) unique values, which exceeds possible index values in $ET", ) end data = arrowvector( deltas, i, nl, fi, de, ded, nothing; dictencode=dictencodenested, dictencodenested=dictencodenested, dictencoding=true, kw..., ) push!( ded, DictEncoding{eltype(data),ET,typeof(data)}( id, data, false, getmetadata(data), ), ) if typeof(encoding.data) <: ChainedVector append!(encoding.data, data) else data2 = ChainedVector([encoding.data, data]) encoding = DictEncoding{eltype(data2),ET,typeof(data2)}( id, data2, false, getmetadata(encoding), ) de[id] = Lockable(encoding) end end end end if meta !== nothing && getmetadata(encoding) !== nothing meta = toidict(merge!(Dict(meta), Dict(getmetadata(encoding)))) elseif getmetadata(encoding) !== nothing meta = getmetadata(encoding) end return DictEncoded(UInt8[], validity, inds, encoding, meta) end @propagate_inbounds function Base.getindex(d::DictEncoded, i::Integer) @boundscheck checkbounds(d, i) @inbounds valid = d.validity[i] !valid && return missing @inbounds idx = d.indices[i] return @inbounds d.encoding[idx + 1] end @propagate_inbounds function Base.setindex!(d::DictEncoded{T}, v, i::Integer) where {T} @boundscheck checkbounds(d, i) if v === missing @inbounds d.validity[i] = false else ix = findfirst(d.encoding.data, v) if ix === nothing push!(d.encoding.data, v) @inbounds d.indices[i] = length(d.encoding.data) - 1 else @inbounds d.indices[i] = ix - 1 end end return v end function Base.copy(x::DictEncoded{T,S}) where {T,S} pool = copy(x.encoding.data) valid = x.validity inds = x.indices refs = copy(inds) @inbounds for i = 1:length(inds) refs[i] = refs[i] + one(S) end return PooledArray( PooledArrays.RefArray(refs), Dict{T,S}(val => i for (i, val) in enumerate(pool)), pool, ) end function compress(Z::Meta.CompressionType.T, comp, x::A) where {A<:DictEncoded} len = length(x) nc = nullcount(x) validity = compress(Z, comp, x.validity) inds = compress(Z, comp, x.indices) return Compressed{Z,A}(x, [validity, inds], len, nc, Compressed[]) end function DataAPI.levels(x::DictEncoded) rp = DataAPI.refpool(x) # may contain missing values Missing <: eltype(rp) || return rp convert(AbstractArray{nonmissingtype(eltype(rp))}, deleteat!(rp, ismissing.(rp))) end function makenodesbuffers!( col::DictEncoded{T,S}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) where {T,S} len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += blen # indices blen = sizeof(S) * len push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) return bufferoffset end DataAPI.refarray(x::DictEncoded{T,S}) where {T,S} = x.indices .+ one(S) DataAPI.refpool(x::DictEncoded) = copy(x.encoding.data) function writebuffer(io, col::DictEncoded, alignment) @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) # write indices n = writearray(io, col.indices) @debug "writing array: col = $(typeof(col.indices)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) return end ================================================ FILE: src/arraytypes/fixedsizelist.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.FixedSizeList An `ArrowVector` where each element is a "fixed size" list of some kind, like a `NTuple{N, T}`. """ struct FixedSizeList{T,A<:AbstractVector} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob validity::ValidityBitmap data::A ℓ::Int metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(l::FixedSizeList) = (l.ℓ,) @propagate_inbounds function Base.getindex(l::FixedSizeList{T}, i::Integer) where {T} @boundscheck checkbounds(l, i) S = Base.nonmissingtype(T) X = ArrowTypes.ArrowKind(ArrowTypes.ArrowType(S)) N = ArrowTypes.getsize(X) Y = ArrowTypes.gettype(X) if X !== T && !(l.validity[i]) return missing else off = (i - 1) * N if X === T && isbitstype(Y) tup = _unsafe_load_tuple(NTuple{N,Y}, l.data, off + 1) else tup = ntuple(j -> l.data[off + j], N) end return ArrowTypes.fromarrow(T, tup) end end function _unsafe_load_tuple( ::Type{NTuple{N,T}}, bytes::Vector{UInt8}, i::Integer, ) where {N,T} x = Ref(bytes, i) y = Ref{NTuple{N,T}}() ArrowTypes._unsafe_cast!(y, x, N) return y[] end @propagate_inbounds function Base.setindex!(l::FixedSizeList{T}, v::T, i::Integer) where {T} @boundscheck checkbounds(l, i) if v === missing @inbounds l.validity[i] = false else N = ArrowTypes.getsize( ArrowTypes.ArrowKind(ArrowTypes.ArrowType(Base.nonmissingtype(T))), ) off = (i - 1) * N foreach(1:N) do j @inbounds l.data[off + j] = v[j] end end return v end # lazy equal-spaced flattener struct ToFixedSizeList{T,N,A} <: AbstractVector{T} data::A # A is AbstractVector of (AbstractVector or AbstractString) end origtype(::ToFixedSizeList{T,N,A}) where {T,N,A} = eltype(A) function ToFixedSizeList(input) NT = ArrowTypes.ArrowKind(Base.nonmissingtype(eltype(input))) # typically NTuple{N, T} return ToFixedSizeList{ArrowTypes.gettype(NT),ArrowTypes.getsize(NT),typeof(input)}( input, ) end Base.IndexStyle(::Type{<:ToFixedSizeList}) = Base.IndexLinear() Base.size(x::ToFixedSizeList{T,N}) where {T,N} = (N * length(x.data),) Base.@propagate_inbounds function Base.getindex( A::ToFixedSizeList{T,N}, i::Integer, ) where {T,N} @boundscheck checkbounds(A, i) a, b = fldmod1(i, N) @inbounds x = A.data[a] return @inbounds x === missing ? ArrowTypes.default(T) : x[b] end # efficient iteration @inline function Base.iterate( A::ToFixedSizeList{T,N}, (i, chunk, chunk_i, len)=(1, 1, 1, length(A)), ) where {T,N} i > len && return nothing @inbounds y = A.data[chunk] @inbounds x = y === missing ? ArrowTypes.default(T) : y[chunk_i] if chunk_i == N chunk += 1 chunk_i = 1 else chunk_i += 1 end return x, (i + 1, chunk, chunk_i, len) end arrowvector(::FixedSizeListKind, x::FixedSizeList, i, nl, fi, de, ded, meta; kw...) = x function arrowvector( ::FixedSizeListKind{N,T}, x, i, nl, fi, de, ded, meta; kw..., ) where {N,T} len = length(x) validity = ValidityBitmap(x) flat = ToFixedSizeList(x) if eltype(flat) == UInt8 data = flat S = origtype(flat) else data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; kw...) S = withmissing(eltype(x), NTuple{N,eltype(data)}) end return FixedSizeList{S,typeof(data)}(UInt8[], validity, data, len, meta) end function compress(Z::Meta.CompressionType.T, comp, x::FixedSizeList{T,A}) where {T,A} len = length(x) nc = nullcount(x) validity = compress(Z, comp, x.validity) buffers = [validity] children = Compressed[] if eltype(A) == UInt8 push!(buffers, compress(Z, comp, x.data)) else push!(children, compress(Z, comp, x.data)) end return Compressed{Z,typeof(x)}(x, buffers, len, nc, children) end function makenodesbuffers!( col::FixedSizeList{T,A}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) where {T,A} len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += blen if eltype(A) === UInt8 blen = ArrowTypes.getsize(ArrowTypes.ArrowKind(Base.nonmissingtype(T))) * len push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) else bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset, alignment) end return bufferoffset end function writebuffer(io, col::FixedSizeList{T,A}, alignment) where {T,A} @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) # write values array if eltype(A) === UInt8 n = writearray(io, UInt8, col.data) @debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) else writebuffer(io, col.data, alignment) end return end ================================================ FILE: src/arraytypes/list.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct Offsets{T<:Union{Int32,Int64}} <: ArrowVector{Tuple{T,T}} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob offsets::Vector{T} end Base.size(o::Offsets) = (length(o.offsets) - 1,) @propagate_inbounds function Base.getindex(o::Offsets, i::Integer) @boundscheck checkbounds(o, i) @inbounds lo = o.offsets[i] + 1 @inbounds hi = o.offsets[i + 1] return lo, hi end """ Arrow.List An `ArrowVector` where each element is a variable sized list of some kind, like an `AbstractVector` or `AbstractString`. """ struct List{T,O,A} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob validity::ValidityBitmap offsets::Offsets{O} data::A ℓ::Int metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(l::List) = (l.ℓ,) @propagate_inbounds function Base.getindex(l::List{T}, i::Integer) where {T} @boundscheck checkbounds(l, i) @inbounds lo, hi = l.offsets[i] S = Base.nonmissingtype(T) K = ArrowTypes.ArrowKind(ArrowTypes.ArrowType(S)) # special-case Base.CodeUnits for ArrowTypes compat if ArrowTypes.isstringtype(K) || S <: Base.CodeUnits if S !== T if S <: Base.CodeUnits return l.validity[i] ? Base.CodeUnits(unsafe_string(pointer(l.data, lo), hi - lo + 1)) : missing else return l.validity[i] ? ArrowTypes.fromarrow(T, pointer(l.data, lo), hi - lo + 1) : missing end else if S <: Base.CodeUnits return Base.CodeUnits(unsafe_string(pointer(l.data, lo), hi - lo + 1)) else return ArrowTypes.fromarrow(T, pointer(l.data, lo), hi - lo + 1) end end elseif S !== T return l.validity[i] ? ArrowTypes.fromarrow(T, view(l.data, lo:hi)) : missing else return ArrowTypes.fromarrow(T, view(l.data, lo:hi)) end end # @propagate_inbounds function Base.setindex!(l::List{T}, v, i::Integer) where {T} # end # internal interface definitions to be able to treat AbstractString/CodeUnits similarly _ncodeunits(x::AbstractString) = ncodeunits(x) _codeunits(x::AbstractString) = codeunits(x) _ncodeunits(x::Base.CodeUnits) = length(x) _codeunits(x::Base.CodeUnits) = x # an AbstractVector version of Iterators.flatten # code based on SentinelArrays.ChainedVector struct ToList{T,stringtype,A,I} <: AbstractVector{T} data::Vector{A} # A is AbstractVector or AbstractString inds::Vector{I} end origtype(::ToList{T,S,A,I}) where {T,S,A,I} = A liststringtype(::Type{ToList{T,S,A,I}}) where {T,S,A,I} = S function liststringtype(::List{T,O,A}) where {T,O,A} ST = Base.nonmissingtype(T) K = ArrowTypes.ArrowKind(ST) return liststringtype(A) || ArrowTypes.isstringtype(K) || ST <: Base.CodeUnits # add the CodeUnits check for ArrowTypes compat for now end liststringtype(T) = false function ToList(input; largelists::Bool=false) AT = eltype(input) ST = Base.nonmissingtype(AT) K = ArrowTypes.ArrowKind(ST) stringtype = ArrowTypes.isstringtype(K) || ST <: Base.CodeUnits # add the CodeUnits check for ArrowTypes compat for now T = stringtype ? UInt8 : eltype(ST) len = stringtype ? _ncodeunits : length data = AT[] I = largelists ? Int64 : Int32 inds = I[0] sizehint!(data, length(input)) sizehint!(inds, length(input)) totalsize = I(0) for x in input if x === missing push!(data, missing) else push!(data, x) totalsize += len(x) if I === Int32 && totalsize > 2147483647 I = Int64 inds = convert(Vector{Int64}, inds) end end push!(inds, totalsize) end return ToList{T,stringtype,AT,I}(data, inds) end Base.IndexStyle(::Type{<:ToList}) = Base.IndexLinear() Base.size(x::ToList{T,S,A,I}) where {T,S,A,I} = (isempty(x.inds) ? zero(I) : x.inds[end],) function Base.pointer(A::ToList{UInt8}, i::Integer) chunk = searchsortedfirst(A.inds, i) chunk = chunk > length(A.inds) ? 1 : (chunk - 1) return pointer(A.data[chunk]) end @inline function index(A::ToList, i::Integer) chunk = searchsortedfirst(A.inds, i) return chunk - 1, i - (@inbounds A.inds[chunk - 1]) end Base.@propagate_inbounds function Base.getindex( A::ToList{T,stringtype}, i::Integer, ) where {T,stringtype} @boundscheck checkbounds(A, i) chunk, ix = index(A, i) @inbounds x = A.data[chunk] return @inbounds stringtype ? _codeunits(x)[ix] : x[ix] end Base.@propagate_inbounds function Base.setindex!( A::ToList{T,stringtype}, v, i::Integer, ) where {T,stringtype} @boundscheck checkbounds(A, i) chunk, ix = index(A, i) @inbounds x = A.data[chunk] if stringtype _codeunits(x)[ix] = v else x[ix] = v end return v end # efficient iteration @inline function Base.iterate(A::ToList{T,stringtype}) where {T,stringtype} length(A) == 0 && return nothing i = 1 chunk = 2 chunk_i = 1 chunk_len = A.inds[chunk] while i > chunk_len chunk += 1 chunk_len = A.inds[chunk] end val = A.data[chunk - 1] x = stringtype ? _codeunits(val)[1] : val[1] # find next valid index i += 1 if i > chunk_len while true chunk += 1 chunk > length(A.inds) && break chunk_len = A.inds[chunk] i <= chunk_len && break end else chunk_i += 1 end return x, (i, chunk, chunk_i, chunk_len, length(A)) end @inline function Base.iterate( A::ToList{T,stringtype}, (i, chunk, chunk_i, chunk_len, len), ) where {T,stringtype} i > len && return nothing @inbounds val = A.data[chunk - 1] @inbounds x = stringtype ? _codeunits(val)[chunk_i] : val[chunk_i] i += 1 if i > chunk_len chunk_i = 1 while true chunk += 1 chunk > length(A.inds) && break @inbounds chunk_len = A.inds[chunk] i <= chunk_len && break end else chunk_i += 1 end return x, (i, chunk, chunk_i, chunk_len, len) end arrowvector(::ListKind, x::List, i, nl, fi, de, ded, meta; kw...) = x function arrowvector(::ListKind, x, i, nl, fi, de, ded, meta; largelists::Bool=false, kw...) len = length(x) validity = ValidityBitmap(x) flat = ToList(x; largelists=largelists) offsets = Offsets(UInt8[], flat.inds) if liststringtype(typeof(flat)) && eltype(flat) == UInt8 # binary or utf8string data = flat T = origtype(flat) else data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; largelists=largelists, kw...) T = withmissing(eltype(x), Vector{eltype(data)}) end return List{T,eltype(flat.inds),typeof(data)}( UInt8[], validity, offsets, data, len, meta, ) end function compress(Z::Meta.CompressionType.T, comp, x::List{T,O,A}) where {T,O,A} len = length(x) nc = nullcount(x) validity = compress(Z, comp, x.validity) offsets = compress(Z, comp, x.offsets.offsets) buffers = [validity, offsets] children = Compressed[] if liststringtype(x) push!(buffers, compress(Z, comp, x.data)) else push!(children, compress(Z, comp, x.data)) end return Compressed{Z,typeof(x)}(x, buffers, len, nc, children) end ================================================ FILE: src/arraytypes/map.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.Map An `ArrowVector` where each element is a "map" of some kind, like a `Dict`. """ struct Map{T,O,A} <: ArrowVector{T} validity::ValidityBitmap offsets::Offsets{O} data::A ℓ::Int metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(l::Map) = (l.ℓ,) @propagate_inbounds function Base.getindex(l::Map{T}, i::Integer) where {T} @boundscheck checkbounds(l, i) @inbounds lo, hi = l.offsets[i] if Base.nonmissingtype(T) !== T return l.validity[i] ? ArrowTypes.fromarrow( T, Dict(x.key => x.value for x in view(l.data, lo:hi)), ) : missing else return ArrowTypes.fromarrow(T, Dict(x.key => x.value for x in view(l.data, lo:hi))) end end keyvalues(KT, ::Missing) = missing keyvalues(KT, x::AbstractDict) = [KT(k, v) for (k, v) in pairs(x)] keyvaluetypes(::Type{NamedTuple{(:key, :value),Tuple{K,V}}}) where {K,V} = (K, V) arrowvector(::MapKind, x::Map, i, nl, fi, de, ded, meta; kw...) = x function arrowvector(::MapKind, x, i, nl, fi, de, ded, meta; largelists::Bool=false, kw...) len = length(x) validity = ValidityBitmap(x) ET = eltype(x) DT = Base.nonmissingtype(ET) KDT, VDT = keytype(DT), valtype(DT) ArrowTypes.concrete_or_concreteunion(KDT) || throw( ArgumentError( "`keytype(d)` must be concrete to serialize map-like `d`, but `keytype(d) == $KDT`", ), ) ArrowTypes.concrete_or_concreteunion(VDT) || throw( ArgumentError( "`valtype(d)` must be concrete to serialize map-like `d`, but `valtype(d) == $VDT`", ), ) KT = KeyValue{KDT,VDT} VT = Vector{KT} T = DT !== ET ? Union{Missing,VT} : VT flat = ToList(T[keyvalues(KT, y) for y in x]; largelists=largelists) offsets = Offsets(UInt8[], flat.inds) data = arrowvector(flat, i, nl + 1, fi, de, ded, nothing; largelists=largelists, kw...) K, V = keyvaluetypes(eltype(data)) return Map{withmissing(ET, Dict{K,V}),eltype(flat.inds),typeof(data)}( validity, offsets, data, len, meta, ) end function compress(Z::Meta.CompressionType.T, comp, x::A) where {A<:Map} len = length(x) nc = nullcount(x) validity = compress(Z, comp, x.validity) offsets = compress(Z, comp, x.offsets.offsets) buffers = [validity, offsets] children = Compressed[] push!(children, compress(Z, comp, x.data)) return Compressed{Z,A}(x, buffers, len, nc, children) end function makenodesbuffers!( col::Union{Map{T,O,A},List{T,O,A}}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) where {T,O,A} len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" # adjust buffer offset, make array buffer bufferoffset += blen blen = sizeof(O) * (len + 1) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) if liststringtype(col) blen = length(col.data) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) else bufferoffset = makenodesbuffers!(col.data, fieldnodes, fieldbuffers, bufferoffset, alignment) end return bufferoffset end function writebuffer(io, col::Union{Map{T,O,A},List{T,O,A}}, alignment) where {T,O,A} @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) # write offsets n = writearray(io, O, col.offsets.offsets) @debug "writing array: col = $(typeof(col.offsets.offsets)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) # write values array if liststringtype(col) n = writearray(io, UInt8, col.data) @debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) else writebuffer(io, col.data, alignment) end return end ================================================ FILE: src/arraytypes/primitive.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.Primitive An `ArrowVector` where each element is a "fixed size" scalar of some kind, like an integer, float, decimal, or time type. """ struct Primitive{T,A} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob validity::ValidityBitmap data::A ℓ::Int64 metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Primitive(::Type{T}, b::Vector{UInt8}, v::ValidityBitmap, data::A, l, meta) where {T,A} = Primitive{T,A}(b, v, data, l, meta) Base.size(p::Primitive) = (p.ℓ,) function Base.copy(p::Primitive{T,A}) where {T,A} if nullcount(p) == 0 && T === eltype(A) return copy(p.data) else return convert(Array, p) end end @propagate_inbounds function Base.getindex(p::Primitive{T}, i::Integer) where {T} @boundscheck checkbounds(p, i) if T >: Missing return @inbounds (p.validity[i] ? ArrowTypes.fromarrow(T, p.data[i]) : missing) else return @inbounds ArrowTypes.fromarrow(T, p.data[i]) end end @propagate_inbounds function Base.setindex!(p::Primitive{T}, v, i::Integer) where {T} @boundscheck checkbounds(p, i) if T >: Missing if v === missing @inbounds p.validity[i] = false else @inbounds p.data[i] = convert(Base.nonmissingtype(T), v) end else @inbounds p.data[i] = convert(Base.nonmissingtype(T), v) end return v end arrowvector(::PrimitiveKind, x::Primitive, i, nl, fi, de, ded, meta; kw...) = x function arrowvector(::PrimitiveKind, x, i, nl, fi, de, ded, meta; kw...) validity = ValidityBitmap(x) return Primitive(eltype(x), UInt8[], validity, x, length(x), meta) end function compress(Z::Meta.CompressionType.T, comp, p::P) where {P<:Primitive} len = length(p) nc = nullcount(p) validity = compress(Z, comp, p.validity) data = compress(Z, comp, p.data) return Compressed{Z,P}(p, [validity, data], len, nc, Compressed[]) end function makenodesbuffers!( col::Primitive{T}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) where {T} len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" # adjust buffer offset, make primitive array buffer bufferoffset += blen blen = len * sizeof(Base.nonmissingtype(T)) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" return bufferoffset + padding(blen, alignment) end function writebuffer(io, col::Primitive{T}, alignment) where {T} @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) n = writearray(io, Base.nonmissingtype(T), col.data) @debug "writing array: col = $(typeof(col.data)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) return end ================================================ FILE: src/arraytypes/struct.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Arrow.Struct An `ArrowVector` where each element is a "struct" of some kind with ordered, named fields, like a `NamedTuple{names, types}` or regular julia `struct`. """ struct Struct{T,S,fnames} <: ArrowVector{T} validity::ValidityBitmap data::S # Tuple of ArrowVector ℓ::Int metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(s::Struct) = (s.ℓ,) isnamedtuple(::Type{<:NamedTuple}) = true isnamedtuple(T) = false istuple(::Type{<:Tuple}) = true istuple(T) = false if isdefined(ArrowTypes, :fromarrowstruct) # https://github.com/apache/arrow-julia/pull/493 @inline function _fromarrowstruct(T::Type, v::Val, x...) return ArrowTypes.fromarrowstruct(T, v, x...) end else @inline function _fromarrowstruct(T::Type, ::Val, x...) return ArrowTypes.fromarrow(T, x...) end end @propagate_inbounds function Base.getindex( s::Struct{T,S,fnames}, i::Integer, ) where {T,S,fnames} @boundscheck checkbounds(s, i) NT = Base.nonmissingtype(T) NT !== T && (s.validity[i] || return missing) vals = ntuple(j -> s.data[j][i], fieldcount(S)) if isnamedtuple(NT) || istuple(NT) return NT(vals) else return _fromarrowstruct(NT, Val{fnames}(), vals...) end end # @propagate_inbounds function Base.setindex!(s::Struct{T}, v::T, i::Integer) where {T} # @boundscheck checkbounds(s, i) # if v === missing # @inbounds s.validity[i] = false # else # NT = Base.nonmissingtype(T) # N = fieldcount(NT) # foreach(1:N) do j # @inbounds s.data[j][i] = getfield(v, j) # end # end # return v # end struct ToStruct{T,i,A} <: AbstractVector{T} data::A # eltype is NamedTuple or some struct end ToStruct(x::A, j::Integer) where {A} = ToStruct{fieldtype(Base.nonmissingtype(eltype(A)), j),j,A}(x) Base.IndexStyle(::Type{<:ToStruct}) = Base.IndexLinear() Base.size(x::ToStruct) = (length(x.data),) Base.@propagate_inbounds function Base.getindex(A::ToStruct{T,j}, i::Integer) where {T,j} @boundscheck checkbounds(A, i) @inbounds x = A.data[i] return x === missing ? ArrowTypes.default(T) : getfield(x, j) end arrowvector(::StructKind, x::Struct, i, nl, fi, de, ded, meta; kw...) = x namedtupletype(::Type{NamedTuple{names,types}}, data) where {names,types} = NamedTuple{names,Tuple{(eltype(x) for x in data)...}} namedtupletype(::Type{T}, data) where {T} = NamedTuple{fieldnames(T),Tuple{(eltype(x) for x in data)...}} namedtupletype(::Type{T}, data) where {T<:Tuple} = NamedTuple{map(Symbol, fieldnames(T)),Tuple{(eltype(x) for x in data)...}} function arrowvector(::StructKind, x, i, nl, fi, de, ded, meta; kw...) len = length(x) validity = ValidityBitmap(x) T = Base.nonmissingtype(eltype(x)) data = Tuple( arrowvector(ToStruct(x, j), i, nl + 1, j, de, ded, nothing; kw...) for j = 1:fieldcount(T) ) NT = namedtupletype(T, data) return Struct{withmissing(eltype(x), NT),typeof(data),fieldnames(NT)}( validity, data, len, meta, ) end function compress(Z::Meta.CompressionType.T, comp, x::A) where {A<:Struct} len = length(x) nc = nullcount(x) validity = compress(Z, comp, x.validity) buffers = [validity] children = Compressed[] for y in x.data push!(children, compress(Z, comp, y)) end return Compressed{Z,A}(x, buffers, len, nc, children) end function makenodesbuffers!( col::Struct{T}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) where {T} len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # validity bitmap blen = nc == 0 ? 0 : bitpackedbytes(len, alignment) push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += blen for child in col.data bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment) end return bufferoffset end function writebuffer(io, col::Struct, alignment) @debug "writebuffer: col = $(typeof(col))" @debug col writebitmap(io, col, alignment) # write values arrays for child in col.data writebuffer(io, child, alignment) end return end ================================================ FILE: src/arraytypes/unions.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Union arrays # need a custom representation of Union types since arrow unions # are ordered, and possibly indirected via separate typeIds array # here, T is Meta.UnionMode.Dense or Meta.UnionMode.Sparse, # typeIds is a NTuple{N, Int32}, and U is a Tuple{...} of the # unioned types struct UnionT{T,typeIds,U} end unionmode(::Type{UnionT{T,typeIds,U}}) where {T,typeIds,U} = T typeids(::Type{UnionT{T,typeIds,U}}) where {T,typeIds,U} = typeIds Base.eltype(::Type{UnionT{T,typeIds,U}}) where {T,typeIds,U} = U uniontypewith(::Type{UnionT{T,typeIds,U}}, ::Type{U2}) where {T,typeIds,U,U2<:Tuple} = UnionT{T,typeIds,U2} ArrowTypes.ArrowKind(::Type{<:UnionT}) = ArrowTypes.UnionKind() # iterate a Julia Union{...} type, producing an array of unioned types function eachunion(U::Union, elems=nothing) if elems === nothing return eachunion(U.b, Type[U.a]) else push!(elems, U.a) return eachunion(U.b, elems) end end function eachunion(T, elems) push!(elems, T) return elems end # produce typeIds, offsets, data tuple for DenseUnion isatypeid(x::T, ::Type{types}) where {T,types} = isatypeid(x, fieldtype(types, 1), types, 1) isatypeid(x::T, ::Type{S}, ::Type{types}, i) where {T,S,types} = x isa S ? i : isatypeid(x, fieldtype(types, i + 1), types, i + 1) """ Arrow.DenseUnion An `ArrowVector` where the type of each element is one of a fixed set of types, meaning its eltype is like a julia `Union{type1, type2, ...}`. An `Arrow.DenseUnion`, in comparison to `Arrow.SparseUnion`, stores elements in a set of arrays, one array per possible type, and an "offsets" array, where each offset element is the index into one of the typed arrays. This allows a sort of "compression", where no extra space is used/allocated to store all the elements. """ struct DenseUnion{T,U,S} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob arrow2::Vector{UInt8} # if arrow blob is compressed, need a 2nd reference for uncompressed offsets bytes typeIds::Vector{UInt8} offsets::Vector{Int32} data::S # Tuple of ArrowVector metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(s::DenseUnion) = size(s.typeIds) nullcount(x::DenseUnion) = 0 # DenseUnion has no validity bitmap; only children do @propagate_inbounds function Base.getindex( s::DenseUnion{T,UnionT{M,typeIds,U}}, i::Integer, ) where {T,M,typeIds,U} @boundscheck checkbounds(s, i) @inbounds typeId = s.typeIds[i] @inbounds off = s.offsets[i] @inbounds x = s.data[typeId + 1][off + 1] return ArrowTypes.fromarrow(fieldtype(U, typeId + 1), x) end # @propagate_inbounds function Base.setindex!(s::DenseUnion{UnionT{T, typeIds, U}}, v, i::Integer) where {T, typeIds, U} # @boundscheck checkbounds(s, i) # @inbounds typeId = s.typeIds[i] # typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds # vtypeId = Int8(typeids[isatypeid(v, U)]) # if typeId == vtypeId # @inbounds off = s.offsets[i] # @inbounds s.data[typeId +1][off + 1] = v # else # throw(ArgumentError("type of item to set $(typeof(v)) must match existing item $(fieldtype(U, typeid))")) # end # return v # end # convenience wrappers for signaling that an array shoudld be written # as with dense/sparse union arrow buffers struct DenseUnionVector{T,U} <: AbstractVector{UnionT{Meta.UnionMode.Dense,nothing,U}} itr::T end DenseUnionVector(x::T) where {T} = DenseUnionVector{T,Tuple{eachunion(eltype(x))...}}(x) Base.IndexStyle(::Type{<:DenseUnionVector}) = Base.IndexLinear() Base.size(x::DenseUnionVector) = (length(x.itr),) Base.iterate(x::DenseUnionVector, st...) = iterate(x.itr, st...) Base.getindex(x::DenseUnionVector, i::Int) = getindex(x.itr, i) function todense(::Type{UnionT{T,typeIds,U}}, x) where {T,typeIds,U} typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds len = length(x) types = Vector{UInt8}(undef, len) offsets = Vector{Int32}(undef, len) data = Tuple( Vector{i == 1 ? Union{Missing,fieldtype(U, i)} : fieldtype(U, i)}(undef, 0) for i = 1:fieldcount(U) ) for (i, y) in enumerate(x) typeid = y === missing ? 0x00 : UInt8(typeids[isatypeid(y, U)]) @inbounds types[i] = typeid @inbounds offsets[i] = length(data[typeid + 1]) push!(data[typeid + 1], y) end return types, offsets, data end struct SparseUnionVector{T,U} <: AbstractVector{UnionT{Meta.UnionMode.Sparse,nothing,U}} itr::T end SparseUnionVector(x::T) where {T} = SparseUnionVector{T,Tuple{eachunion(eltype(x))...}}(x) Base.IndexStyle(::Type{<:SparseUnionVector}) = Base.IndexLinear() Base.size(x::SparseUnionVector) = (length(x.itr),) Base.iterate(x::SparseUnionVector, st...) = iterate(x.itr, st...) Base.getindex(x::SparseUnionVector, i::Int) = getindex(x.itr, i) # sparse union child array producer # for sparse unions, we split the parent array into # N children arrays, each having the same length as the parent # but with one child array per unioned type; each child # should include the elements from parent of its type # and other elements can be missing/default function sparsetypeids(::Type{UnionT{T,typeIds,U}}, x) where {T,typeIds,U} typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds len = length(x) types = Vector{UInt8}(undef, len) for (i, y) in enumerate(x) typeid = y === missing ? 0x00 : UInt8(typeids[isatypeid(y, U)]) @inbounds types[i] = typeid end return types end struct ToSparseUnion{T,A} <: AbstractVector{T} data::A end ToSparseUnion(::Type{T}, data::A) where {T,A} = ToSparseUnion{T,A}(data) Base.IndexStyle(::Type{<:ToSparseUnion}) = Base.IndexLinear() Base.size(x::ToSparseUnion) = (length(x.data),) Base.@propagate_inbounds function Base.getindex(A::ToSparseUnion{T}, i::Integer) where {T} @boundscheck checkbounds(A, i) @inbounds x = A.data[i] return @inbounds x isa T ? x : ArrowTypes.default(T) end function compress(Z::Meta.CompressionType.T, comp, x::A) where {A<:DenseUnion} len = length(x) nc = nullcount(x) typeIds = compress(Z, comp, x.typeIds) offsets = compress(Z, comp, x.offsets) buffers = [typeIds, offsets] children = Compressed[] for y in x.data push!(children, compress(Z, comp, y)) end return Compressed{Z,A}(x, buffers, len, nc, children) end """ Arrow.SparseUnion An `ArrowVector` where the type of each element is one of a fixed set of types, meaning its eltype is like a julia `Union{type1, type2, ...}`. An `Arrow.SparseUnion`, in comparison to `Arrow.DenseUnion`, stores elements in a set of arrays, one array per possible type, and each typed array has the same length as the full array. This ends up with "wasted" space, since only one slot among the typed arrays is valid per full array element, but can allow for certain optimizations when each typed array has the same length. """ struct SparseUnion{T,U,S} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob typeIds::Vector{UInt8} data::S # Tuple of ArrowVector metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(s::SparseUnion) = size(s.typeIds) nullcount(x::SparseUnion) = 0 @propagate_inbounds function Base.getindex( s::SparseUnion{T,UnionT{M,typeIds,U}}, i::Integer, ) where {T,M,typeIds,U} @boundscheck checkbounds(s, i) @inbounds typeId = s.typeIds[i] @inbounds x = s.data[typeId + 1][i] return ArrowTypes.fromarrow(fieldtype(U, typeId + 1), x) end # @propagate_inbounds function Base.setindex!(s::SparseUnion{UnionT{T, typeIds, U}}, v, i::Integer) where {T, typeIds, U} # @boundscheck checkbounds(s, i) # typeids = typeIds === nothing ? (0:(fieldcount(U) - 1)) : typeIds # vtypeId = Int8(typeids[isatypeid(v, U)]) # @inbounds s.typeIds[i] = vtypeId # @inbounds s.data[vtypeId + 1][i] = v # return v # end arrowvector(U::Union, x, i, nl, fi, de, ded, meta; denseunions::Bool=true, kw...) = arrowvector( denseunions ? DenseUnionVector(x) : SparseUnionVector(x), i, nl, fi, de, ded, meta; denseunions=denseunions, kw..., ) arrowvector( ::UnionKind, x::Union{DenseUnion,SparseUnion}, i, nl, fi, de, ded, meta; kw..., ) = x function arrowvector(::UnionKind, x, i, nl, fi, de, ded, meta; kw...) UT = eltype(x) if unionmode(UT) == Meta.UnionMode.Dense x = x isa DenseUnionVector ? x.itr : x typeids, offsets, data = todense(UT, x) data2 = map( y -> arrowvector(y[2], i, nl + 1, y[1], de, ded, nothing; kw...), enumerate(data), ) UT2 = uniontypewith(UT, Tuple{(eltype(x) for x in data2)...}) return DenseUnion{Union{(eltype(x) for x in data2)...},UT2,typeof(data2)}( UInt8[], UInt8[], typeids, offsets, data2, meta, ) else x = x isa SparseUnionVector ? x.itr : x typeids = sparsetypeids(UT, x) data3 = Tuple( arrowvector( ToSparseUnion(fieldtype(eltype(UT), j), x), i, nl + 1, j, de, ded, nothing; kw..., ) for j = 1:fieldcount(eltype(UT)) ) UT2 = uniontypewith(UT, Tuple{(eltype(x) for x in data3)...}) return SparseUnion{Union{(eltype(x) for x in data3)...},UT2,typeof(data3)}( UInt8[], typeids, data3, meta, ) end end function compress(Z::Meta.CompressionType.T, comp, x::A) where {A<:SparseUnion} len = length(x) nc = nullcount(x) typeIds = compress(Z, comp, x.typeIds) buffers = [typeIds] children = Compressed[] for y in x.data push!(children, compress(Z, comp, y)) end return Compressed{Z,A}(x, buffers, len, nc, children) end function makenodesbuffers!( col::Union{DenseUnion,SparseUnion}, fieldnodes, fieldbuffers, bufferoffset, alignment, ) len = length(col) nc = nullcount(col) push!(fieldnodes, FieldNode(len, nc)) @debug "made field node: nodeidx = $(length(fieldnodes)), col = $(typeof(col)), len = $(fieldnodes[end].length), nc = $(fieldnodes[end].null_count)" # typeIds buffer push!(fieldbuffers, Buffer(bufferoffset, len)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(len, alignment) if col isa DenseUnion # offsets buffer blen = sizeof(Int32) * len push!(fieldbuffers, Buffer(bufferoffset, blen)) @debug "made field buffer: bufferidx = $(length(fieldbuffers)), offset = $(fieldbuffers[end].offset), len = $(fieldbuffers[end].length), padded = $(padding(fieldbuffers[end].length, alignment))" bufferoffset += padding(blen, alignment) end for child in col.data bufferoffset = makenodesbuffers!(child, fieldnodes, fieldbuffers, bufferoffset, alignment) end return bufferoffset end function writebuffer(io, col::Union{DenseUnion,SparseUnion}, alignment) @debug "writebuffer: col = $(typeof(col))" @debug col # typeIds buffer n = writearray(io, UInt8, col.typeIds) @debug "writing array: col = $(typeof(col.typeIds)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) if col isa DenseUnion n = writearray(io, Int32, col.offsets) @debug "writing array: col = $(typeof(col.offsets)), n = $n, padded = $(padding(n, alignment))" writezeros(io, paddinglength(n, alignment)) end for child in col.data writebuffer(io, child, alignment) end return end ================================================ FILE: src/arraytypes/views.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct ViewElement length::Int32 prefix::Int32 bufindex::Int32 offset::Int32 end """ Arrow.View An `ArrowVector` where each element is a variable sized list of some kind, like an `AbstractVector` or `AbstractString`. """ struct View{T} <: ArrowVector{T} arrow::Vector{UInt8} # need to hold a reference to arrow memory blob validity::ValidityBitmap data::Vector{ViewElement} inline::Vector{UInt8} # `data` field reinterpreted as a byte array buffers::Vector{Vector{UInt8}} # holds non-inlined data ℓ::Int metadata::Union{Nothing,Base.ImmutableDict{String,String}} end Base.size(l::View) = (l.ℓ,) @propagate_inbounds function Base.getindex(l::View{T}, i::Integer) where {T} @boundscheck checkbounds(l, i) @inbounds v = l.data[i] S = Base.nonmissingtype(T) if S <: Base.CodeUnits # BinaryView return !l.validity[i] ? missing : v.length < 13 ? Base.CodeUnits( StringView( @view l.inline[(((i - 1) * 16) + 5):(((i - 1) * 16) + 5 + v.length - 1)] ), ) : Base.CodeUnits( StringView( @view l.buffers[v.bufindex + 1][(v.offset + 1):(v.offset + v.length)] ), ) else # Utf8View return !l.validity[i] ? missing : v.length < 13 ? ArrowTypes.fromarrow( T, StringView( @view l.inline[(((i - 1) * 16) + 5):(((i - 1) * 16) + 5 + v.length - 1)] ), ) : ArrowTypes.fromarrow( T, StringView( @view l.buffers[v.bufindex + 1][(v.offset + 1):(v.offset + v.length)] ), ) end end # @propagate_inbounds function Base.setindex!(l::List{T}, v, i::Integer) where {T} # end ================================================ FILE: src/eltypes.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Given a flatbuffers metadata type definition (a Field instance from Schema.fbs), translate to the appropriate Julia storage eltype """ function juliaeltype end finaljuliatype(T) = T finaljuliatype(::Type{Missing}) = Missing finaljuliatype(::Type{Union{T,Missing}}) where {T} = Union{Missing,finaljuliatype(T)} """ Given a FlatBuffers.Builder and a Julia column or column eltype, Write the field.type flatbuffer definition of the eltype """ function arrowtype end arrowtype(b, col::AbstractVector{T}) where {T} = arrowtype(b, maybemissing(T)) arrowtype(b, col::DictEncoded) = arrowtype(b, col.encoding.data) arrowtype(b, col::Compressed) = arrowtype(b, col.data) function juliaeltype(f::Meta.Field, ::Nothing, convert::Bool) T = juliaeltype(f, convert) return convert ? finaljuliatype(T) : T end function juliaeltype(f::Meta.Field, meta::AbstractDict{String,String}, convert::Bool) TT = juliaeltype(f, convert) !convert && return TT T = finaljuliatype(TT) if haskey(meta, "ARROW:extension:name") typename = meta["ARROW:extension:name"] metadata = get(meta, "ARROW:extension:metadata", "") JT = ArrowTypes.JuliaType(Val(Symbol(typename)), maybemissing(TT), metadata) if JT !== nothing return f.nullable ? Union{JT,Missing} : JT else @warn "unsupported ARROW:extension:name type: \"$typename\", arrow type = $TT" maxlog = 1 _id = hash((:juliaeltype, typename, TT)) end end return something(TT, T) end function juliaeltype(f::Meta.Field, convert::Bool) T = juliaeltype(f, f.type, convert) return f.nullable ? Union{T,Missing} : T end juliaeltype(f::Meta.Field, ::Meta.Null, convert) = Missing function arrowtype(b, ::Type{Missing}) Meta.nullStart(b) return Meta.Null, Meta.nullEnd(b), nothing end function juliaeltype(f::Meta.Field, int::Meta.Int, convert) if int.is_signed if int.bitWidth == 8 Int8 elseif int.bitWidth == 16 Int16 elseif int.bitWidth == 32 Int32 elseif int.bitWidth == 64 Int64 elseif int.bitWidth == 128 Int128 else error("$int is not valid arrow type metadata") end else if int.bitWidth == 8 UInt8 elseif int.bitWidth == 16 UInt16 elseif int.bitWidth == 32 UInt32 elseif int.bitWidth == 64 UInt64 elseif int.bitWidth == 128 UInt128 else error("$int is not valid arrow type metadata") end end end function arrowtype(b, ::Type{T}) where {T<:Integer} Meta.intStart(b) Meta.intAddBitWidth(b, Int32(8 * sizeof(T))) Meta.intAddIsSigned(b, T <: Signed) return Meta.Int, Meta.intEnd(b), nothing end # primitive types function juliaeltype(f::Meta.Field, fp::Meta.FloatingPoint, convert) if fp.precision == Meta.Precision.HALF Float16 elseif fp.precision == Meta.Precision.SINGLE Float32 elseif fp.precision == Meta.Precision.DOUBLE Float64 end end function arrowtype(b, ::Type{T}) where {T<:AbstractFloat} Meta.floatingPointStart(b) Meta.floatingPointAddPrecision( b, T === Float16 ? Meta.Precision.HALF : T === Float32 ? Meta.Precision.SINGLE : Meta.Precision.DOUBLE, ) return Meta.FloatingPoint, Meta.floatingPointEnd(b), nothing end juliaeltype(f::Meta.Field, b::Union{Meta.Utf8,Meta.LargeUtf8,Meta.Utf8View}, convert) = String datasizeof(x) = sizeof(x) datasizeof(x::AbstractVector) = sum(datasizeof, x) juliaeltype( f::Meta.Field, b::Union{Meta.Binary,Meta.LargeBinary,Meta.BinaryView}, convert, ) = Base.CodeUnits juliaeltype(f::Meta.Field, x::Meta.FixedSizeBinary, convert) = NTuple{Int(x.byteWidth),UInt8} # arggh! Base.write(io::IO, x::NTuple{N,T}) where {N,T} = sum(y -> Base.write(io, y), x) juliaeltype(f::Meta.Field, x::Meta.Bool, convert) = Bool function arrowtype(b, ::Type{Bool}) Meta.boolStart(b) return Meta.Bool, Meta.boolEnd(b), nothing end struct Decimal{P,S,T} value::T # only Int128 or Int256 end Base.zero(::Type{Decimal{P,S,T}}) where {P,S,T} = Decimal{P,S,T}(T(0)) ==(a::Decimal{P,S,T}, b::Decimal{P,S,T}) where {P,S,T} = ==(a.value, b.value) Base.isequal(a::Decimal{P,S,T}, b::Decimal{P,S,T}) where {P,S,T} = isequal(a.value, b.value) function juliaeltype(f::Meta.Field, x::Meta.Decimal, convert) return Decimal{x.precision,x.scale,x.bitWidth == 256 ? Int256 : Int128} end ArrowTypes.ArrowKind(::Type{<:Decimal}) = PrimitiveKind() function arrowtype(b, ::Type{Decimal{P,S,T}}) where {P,S,T} Meta.decimalStart(b) Meta.decimalAddPrecision(b, Int32(P)) Meta.decimalAddScale(b, Int32(S)) Meta.decimalAddBitWidth(b, Int32(T == Int256 ? 256 : 128)) return Meta.Decimal, Meta.decimalEnd(b), nothing end Base.write(io::IO, x::Decimal) = Base.write(io, x.value) abstract type ArrowTimeType end Base.write(io::IO, x::ArrowTimeType) = Base.write(io, x.x) ArrowTypes.ArrowKind(::Type{<:ArrowTimeType}) = PrimitiveKind() struct Date{U,T} <: ArrowTimeType x::T end const DATE = Date{Meta.DateUnit.DAY,Int32} Base.zero(::Type{Date{U,T}}) where {U,T} = Date{U,T}(T(0)) storagetype(::Type{Date{U,T}}) where {U,T} = T bitwidth(x::Meta.DateUnit.T) = x == Meta.DateUnit.DAY ? Int32 : Int64 Date{Meta.DateUnit.DAY}(days) = DATE(Int32(days)) Date{Meta.DateUnit.MILLISECOND}(ms) = Date{Meta.DateUnit.MILLISECOND,Int64}(Int64(ms)) juliaeltype(f::Meta.Field, x::Meta.Date, convert) = Date{x.unit,bitwidth(x.unit)} finaljuliatype(::Type{DATE}) = Dates.Date Base.convert(::Type{Dates.Date}, x::DATE) = Dates.Date(Dates.UTD(Int64(x.x + UNIX_EPOCH_DATE))) finaljuliatype(::Type{Date{Meta.DateUnit.MILLISECOND,Int64}}) = Dates.DateTime Base.convert(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND,Int64}) = Dates.DateTime(Dates.UTM(Int64(x.x + UNIX_EPOCH_DATETIME))) function arrowtype(b, ::Type{Date{U,T}}) where {U,T} Meta.dateStart(b) Meta.dateAddUnit(b, U) return Meta.Date, Meta.dateEnd(b), nothing end const UNIX_EPOCH_DATE = Dates.value(Dates.Date(1970)) Base.convert(::Type{DATE}, x::Dates.Date) = DATE(Int32(Dates.value(x) - UNIX_EPOCH_DATE)) const UNIX_EPOCH_DATETIME = Dates.value(Dates.DateTime(1970)) Base.convert(::Type{Date{Meta.DateUnit.MILLISECOND,Int64}}, x::Dates.DateTime) = Date{Meta.DateUnit.MILLISECOND,Int64}(Int64(Dates.value(x) - UNIX_EPOCH_DATETIME)) ArrowTypes.ArrowType(::Type{Dates.Date}) = DATE ArrowTypes.toarrow(x::Dates.Date) = convert(DATE, x) const DATE_SYMBOL = Symbol("JuliaLang.Date") ArrowTypes.arrowname(::Type{Dates.Date}) = DATE_SYMBOL ArrowTypes.JuliaType(::Val{DATE_SYMBOL}, S) = Dates.Date ArrowTypes.fromarrow(::Type{Dates.Date}, x::DATE) = convert(Dates.Date, x) ArrowTypes.default(::Type{Dates.Date}) = Dates.Date(1, 1, 1) struct Time{U,T} <: ArrowTimeType x::T end Base.zero(::Type{Time{U,T}}) where {U,T} = Time{U,T}(T(0)) const TIME = Time{Meta.TimeUnit.NANOSECOND,Int64} bitwidth(x::Meta.TimeUnit.T) = x == Meta.TimeUnit.SECOND || x == Meta.TimeUnit.MILLISECOND ? Int32 : Int64 Time{U}(x) where {U<:Meta.TimeUnit.T} = Time{U,bitwidth(U)}(bitwidth(U)(x)) storagetype(::Type{Time{U,T}}) where {U,T} = T juliaeltype(f::Meta.Field, x::Meta.Time, convert) = Time{x.unit,bitwidth(x.unit)} finaljuliatype(::Type{<:Time}) = Dates.Time periodtype(U::Meta.TimeUnit.T) = U === Meta.TimeUnit.SECOND ? Dates.Second : U === Meta.TimeUnit.MILLISECOND ? Dates.Millisecond : U === Meta.TimeUnit.MICROSECOND ? Dates.Microsecond : Dates.Nanosecond Base.convert(::Type{Dates.Time}, x::Time{U,T}) where {U,T} = Dates.Time(Dates.Nanosecond(Dates.tons(periodtype(U)(x.x)))) function arrowtype(b, ::Type{Time{U,T}}) where {U,T} Meta.timeStart(b) Meta.timeAddUnit(b, U) Meta.timeAddBitWidth(b, Int32(8 * sizeof(T))) return Meta.Time, Meta.timeEnd(b), nothing end Base.convert(::Type{TIME}, x::Dates.Time) = TIME(Dates.value(x)) ArrowTypes.ArrowType(::Type{Dates.Time}) = TIME ArrowTypes.toarrow(x::Dates.Time) = convert(TIME, x) const TIME_SYMBOL = Symbol("JuliaLang.Time") ArrowTypes.arrowname(::Type{Dates.Time}) = TIME_SYMBOL ArrowTypes.JuliaType(::Val{TIME_SYMBOL}, S) = Dates.Time ArrowTypes.fromarrow(::Type{Dates.Time}, x::Arrow.Time) = convert(Dates.Time, x) ArrowTypes.default(::Type{Dates.Time}) = Dates.Time(1, 1, 1) struct Timestamp{U,TZ} <: ArrowTimeType x::Int64 end Base.zero(::Type{Timestamp{U,T}}) where {U,T} = Timestamp{U,T}(Int64(0)) function juliaeltype(f::Meta.Field, x::Meta.Timestamp, convert) return Timestamp{x.unit,x.timezone === nothing ? nothing : Symbol(x.timezone)} end const DATETIME = Timestamp{Meta.TimeUnit.MILLISECOND,nothing} finaljuliatype(::Type{Timestamp{U,TZ}}) where {U,TZ} = ZonedDateTime finaljuliatype(::Type{Timestamp{U,nothing}}) where {U} = DateTime @noinline warntimestamp(U, T) = @warn "automatically converting Arrow.Timestamp with precision = $U to `$T` which only supports millisecond precision; conversion may be lossy; to avoid converting, pass `Arrow.Table(source; convert=false)" maxlog = 1 _id = hash((:warntimestamp, U, T)) function Base.convert(::Type{ZonedDateTime}, x::Timestamp{U,TZ}) where {U,TZ} (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, ZonedDateTime) return ZonedDateTime( Dates.DateTime( Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME)), ), TimeZone(String(TZ)); from_utc=true, ) end function Base.convert(::Type{DateTime}, x::Timestamp{U,nothing}) where {U} (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, DateTime) return Dates.DateTime( Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME)), ) end Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND,TZ}}, x::ZonedDateTime) where {TZ} = Timestamp{Meta.TimeUnit.MILLISECOND,TZ}( Int64(Dates.value(DateTime(x, UTC)) - UNIX_EPOCH_DATETIME), ) Base.convert(::Type{Timestamp{Meta.TimeUnit.MILLISECOND,nothing}}, x::DateTime) = Timestamp{Meta.TimeUnit.MILLISECOND,nothing}( Int64(Dates.value(x) - UNIX_EPOCH_DATETIME), ) function arrowtype(b, ::Type{Timestamp{U,TZ}}) where {U,TZ} tz = TZ !== nothing ? FlatBuffers.createstring!(b, String(TZ)) : FlatBuffers.UOffsetT(0) Meta.timestampStart(b) Meta.timestampAddUnit(b, U) Meta.timestampAddTimezone(b, tz) return Meta.Timestamp, Meta.timestampEnd(b), nothing end ArrowTypes.ArrowType(::Type{Dates.DateTime}) = DATETIME ArrowTypes.toarrow(x::Dates.DateTime) = convert(DATETIME, x) const DATETIME_SYMBOL = Symbol("JuliaLang.DateTime") ArrowTypes.arrowname(::Type{Dates.DateTime}) = DATETIME_SYMBOL ArrowTypes.JuliaType(::Val{DATETIME_SYMBOL}, S) = Dates.DateTime ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Timestamp) = convert(Dates.DateTime, x) ArrowTypes.fromarrow(::Type{Dates.DateTime}, x::Date{Meta.DateUnit.MILLISECOND,Int64}) = convert(Dates.DateTime, x) ArrowTypes.default(::Type{Dates.DateTime}) = Dates.DateTime(1, 1, 1, 1, 1, 1) ArrowTypes.ArrowType(::Type{ZonedDateTime}) = Timestamp ArrowTypes.toarrow(x::ZonedDateTime) = convert(Timestamp{Meta.TimeUnit.MILLISECOND,Symbol(x.timezone)}, x) const ZONEDDATETIME_SYMBOL = Symbol("JuliaLang.ZonedDateTime-UTC") ArrowTypes.arrowname(::Type{ZonedDateTime}) = ZONEDDATETIME_SYMBOL ArrowTypes.JuliaType(::Val{ZONEDDATETIME_SYMBOL}, S) = ZonedDateTime ArrowTypes.fromarrow(::Type{ZonedDateTime}, x::Timestamp) = convert(ZonedDateTime, x) ArrowTypes.default(::Type{TimeZones.ZonedDateTime}) = TimeZones.ZonedDateTime(1, 1, 1, 1, 1, 1, TimeZones.tz"UTC") # Backwards compatibility: older versions of Arrow saved ZonedDateTime's with this metdata: const OLD_ZONEDDATETIME_SYMBOL = Symbol("JuliaLang.ZonedDateTime") # and stored the local time instead of the UTC time. struct LocalZonedDateTime end ArrowTypes.JuliaType(::Val{OLD_ZONEDDATETIME_SYMBOL}, S) = LocalZonedDateTime function ArrowTypes.fromarrow(::Type{LocalZonedDateTime}, x::Timestamp{U,TZ}) where {U,TZ} (U === Meta.TimeUnit.MICROSECOND || U == Meta.TimeUnit.NANOSECOND) && warntimestamp(U, ZonedDateTime) return ZonedDateTime( Dates.DateTime( Dates.UTM(Int64(Dates.toms(periodtype(U)(x.x)) + UNIX_EPOCH_DATETIME)), ), TimeZone(String(TZ)), ) end """ Arrow.ToTimestamp(x::AbstractVector{ZonedDateTime}) Wrapper array that provides a more efficient encoding of `ZonedDateTime` elements to the arrow format. In the arrow format, timestamp columns with timezone information are encoded as the arrow equivalent of a Julia type parameter, meaning an entire column _should_ have elements all with the same timezone. If a `ZonedDateTime` column is passed to `Arrow.write`, for correctness, it must scan each element to check each timezone. `Arrow.ToTimestamp` provides a "bypass" of this process by encoding the timezone of the first element of the `AbstractVector{ZonedDateTime}`, which in turn allows `Arrow.write` to avoid costly checking/conversion and can encode the `ZonedDateTime` as `Arrow.Timestamp` directly. """ struct ToTimestamp{A,TZ} <: AbstractVector{Timestamp{Meta.TimeUnit.MILLISECOND,TZ}} data::A # AbstractVector{ZonedDateTime} end ToTimestamp(x::A) where {A<:AbstractVector{ZonedDateTime}} = ToTimestamp{A,Symbol(x[1].timezone)}(x) Base.IndexStyle(::Type{<:ToTimestamp}) = Base.IndexLinear() Base.size(x::ToTimestamp) = (length(x.data),) Base.eltype(::Type{ToTimestamp{A,TZ}}) where {A,TZ} = Timestamp{Meta.TimeUnit.MILLISECOND,TZ} Base.getindex(x::ToTimestamp{A,TZ}, i::Integer) where {A,TZ} = convert(Timestamp{Meta.TimeUnit.MILLISECOND,TZ}, getindex(x.data, i)) struct Interval{U,T} <: ArrowTimeType x::T end Base.zero(::Type{Interval{U,T}}) where {U,T} = Interval{U,T}(T(0)) bitwidth(x::Meta.IntervalUnit.T) = x == Meta.IntervalUnit.YEAR_MONTH ? Int32 : Int64 Interval{Meta.IntervalUnit.YEAR_MONTH}(x) = Interval{Meta.IntervalUnit.YEAR_MONTH,Int32}(Int32(x)) Interval{Meta.IntervalUnit.DAY_TIME}(x) = Interval{Meta.IntervalUnit.DAY_TIME,Int64}(Int64(x)) function juliaeltype(f::Meta.Field, x::Meta.Interval, convert) return Interval{x.unit,bitwidth(x.unit)} end function arrowtype(b, ::Type{Interval{U,T}}) where {U,T} Meta.intervalStart(b) Meta.intervalAddUnit(b, U) return Meta.Interval, Meta.intervalEnd(b), nothing end struct Duration{U} <: ArrowTimeType x::Int64 end Base.zero(::Type{Duration{U}}) where {U} = Duration{U}(Int64(0)) function juliaeltype(f::Meta.Field, x::Meta.Duration, convert) return Duration{x.unit} end finaljuliatype(::Type{Duration{U}}) where {U} = periodtype(U) Base.convert(::Type{P}, x::Duration{U}) where {P<:Dates.Period,U} = P(periodtype(U)(x.x)) function arrowtype(b, ::Type{Duration{U}}) where {U} Meta.durationStart(b) Meta.durationAddUnit(b, U) return Meta.Duration, Meta.durationEnd(b), nothing end arrowtype(b, ::Type{P}) where {P<:Dates.Period} = arrowtype(b, Duration{arrowperiodtype(P)}) arrowperiodtype(P) = Meta.TimeUnit.SECOND arrowperiodtype(::Type{Dates.Millisecond}) = Meta.TimeUnit.MILLISECOND arrowperiodtype(::Type{Dates.Microsecond}) = Meta.TimeUnit.MICROSECOND arrowperiodtype(::Type{Dates.Nanosecond}) = Meta.TimeUnit.NANOSECOND Base.convert(::Type{Duration{U}}, x::Dates.Period) where {U} = Duration{U}(Dates.value(periodtype(U)(x))) ArrowTypes.ArrowType(::Type{P}) where {P<:Dates.Period} = Duration{arrowperiodtype(P)} ArrowTypes.toarrow(x::P) where {P<:Dates.Period} = convert(Duration{arrowperiodtype(P)}, x) const PERIOD_SYMBOL = Symbol("JuliaLang.Dates.Period") ArrowTypes.arrowname(::Type{P}) where {P<:Dates.Period} = PERIOD_SYMBOL ArrowTypes.JuliaType(::Val{PERIOD_SYMBOL}, ::Type{Duration{U}}) where {U} = periodtype(U) ArrowTypes.fromarrow(::Type{P}, x::Duration{U}) where {P<:Dates.Period,U} = convert(P, x) # nested types; call juliaeltype recursively on nested children function juliaeltype( f::Meta.Field, list::Union{Meta.List,Meta.LargeList,Meta.ListView,Meta.LargeListView}, convert, ) return Vector{juliaeltype(f.children[1], buildmetadata(f.children[1]), convert)} end # arrowtype will call fieldoffset recursively for children function arrowtype(b, x::List{T,O,A}) where {T,O,A} if liststringtype(x) if T <: AbstractString || T <: Union{AbstractString,Missing} if O == Int32 Meta.utf8Start(b) return Meta.Utf8, Meta.utf8End(b), nothing else # if O == Int64 Meta.largUtf8Start(b) return Meta.LargeUtf8, Meta.largUtf8End(b), nothing end else # if Base.CodeUnits if O == Int32 Meta.binaryStart(b) return Meta.Binary, Meta.binaryEnd(b), nothing else # if O == Int64 Meta.largeBinaryStart(b) return Meta.LargeBinary, Meta.largeBinaryEnd(b), nothing end end else children = [fieldoffset(b, "", x.data)] if O == Int32 Meta.listStart(b) return Meta.List, Meta.listEnd(b), children else Meta.largeListStart(b) return Meta.LargeList, Meta.largeListEnd(b), children end end end function juliaeltype(f::Meta.Field, list::Meta.FixedSizeList, convert) type = juliaeltype(f.children[1], buildmetadata(f.children[1]), convert) return NTuple{Int(list.listSize),type} end function arrowtype(b, x::FixedSizeList{T,A}) where {T,A} N = ArrowTypes.getsize( ArrowTypes.ArrowKind(ArrowTypes.ArrowType(Base.nonmissingtype(T))), ) if eltype(A) == UInt8 Meta.fixedSizeBinaryStart(b) Meta.fixedSizeBinaryAddByteWidth(b, Int32(N)) return Meta.FixedSizeBinary, Meta.fixedSizeBinaryEnd(b), nothing else children = [fieldoffset(b, "", x.data)] Meta.fixedSizeListStart(b) Meta.fixedSizeListAddListSize(b, Int32(N)) return Meta.FixedSizeList, Meta.fixedSizeListEnd(b), children end end function juliaeltype(f::Meta.Field, map::Meta.Map, convert) K = juliaeltype( f.children[1].children[1], buildmetadata(f.children[1].children[1]), convert, ) V = juliaeltype( f.children[1].children[2], buildmetadata(f.children[1].children[2]), convert, ) return Dict{K,V} end function arrowtype(b, x::Map) children = [fieldoffset(b, "entries", x.data)] Meta.mapStart(b) return Meta.Map, Meta.mapEnd(b), children end struct KeyValue{K,V} key::K value::V end keyvalueK(::Type{KeyValue{K,V}}) where {K,V} = K keyvalueV(::Type{KeyValue{K,V}}) where {K,V} = V Base.length(kv::KeyValue) = 1 Base.iterate(kv::KeyValue, st=1) = st === nothing ? nothing : (kv, nothing) ArrowTypes.default(::Type{KeyValue{K,V}}) where {K,V} = KeyValue(default(K), default(V)) function arrowtype(b, ::Type{KeyValue{K,V}}) where {K,V} children = [fieldoffset(b, "key", K), fieldoffset(b, "value", V)] Meta.structStart(b) return Meta.Struct, Meta.structEnd(b), children end function juliaeltype(f::Meta.Field, list::Meta.Struct, convert) names = Tuple(Symbol(x.name) for x in f.children) types = Tuple(juliaeltype(x, buildmetadata(x), convert) for x in f.children) return NamedTuple{names,Tuple{types...}} end function arrowtype(b, x::Struct{T,S}) where {T,S} names = fieldnames(Base.nonmissingtype(T)) children = [fieldoffset(b, names[i], x.data[i]) for i = 1:length(names)] Meta.structStart(b) return Meta.Struct, Meta.structEnd(b), children end # Unions function UnionT(f::Meta.Field, convert) typeids = f.type.typeIds === nothing ? nothing : Tuple(Int(x) for x in f.type.typeIds) UT = UnionT{ f.type.mode, typeids, Tuple{(juliaeltype(x, buildmetadata(x), convert) for x in f.children)...}, } return UT end juliaeltype(f::Meta.Field, u::Meta.Union, convert) = Union{(juliaeltype(x, buildmetadata(x), convert) for x in f.children)...} function arrowtype( b, x::Union{DenseUnion{S,UnionT{T,typeIds,U}},SparseUnion{S,UnionT{T,typeIds,U}}}, ) where {S,T,typeIds,U} if typeIds !== nothing Meta.unionStartTypeIdsVector(b, length(typeIds)) for id in Iterators.reverse(typeIds) FlatBuffers.prepend!(b, id) end TI = FlatBuffers.endvector!(b, length(typeIds)) end children = [fieldoffset(b, "", x.data[i]) for i = 1:fieldcount(U)] Meta.unionStart(b) Meta.unionAddMode(b, T) if typeIds !== nothing Meta.unionAddTypeIds(b, TI) end return Meta.Union, Meta.unionEnd(b), children end ================================================ FILE: src/metadata/File.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct Footer <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Footer) = (:version, :schema, :dictionaries, :recordBatches, :custom_metadata) function Base.getproperty(x::Footer, field::Symbol) if field === :version o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion.T) return MetadataVersion.V1 elseif field === :schema o = FlatBuffers.offset(x, 6) if o != 0 y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x)) return FlatBuffers.init(Schema, FlatBuffers.bytes(x), y) end elseif field === :dictionaries o = FlatBuffers.offset(x, 8) if o != 0 return FlatBuffers.Array{Block}(x, o) end elseif field === :recordBatches o = FlatBuffers.offset(x, 10) if o != 0 return FlatBuffers.Array{Block}(x, o) end elseif field === :custom_metadata o = FlatBuffers.offset(x, 12) if o != 0 return FlatBuffers.Array{KeyValue}(x, o) end end return nothing end footerStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 4) footerAddVersion(b::FlatBuffers.Builder, version::MetadataVersion.T) = FlatBuffers.prependslot!(b, 0, version, 0) footerAddSchema(b::FlatBuffers.Builder, schema::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, schema, 0) footerAddDictionaries(b::FlatBuffers.Builder, dictionaries::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, dictionaries, 0) footerStartDictionariesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8) footerAddRecordBatches(b::FlatBuffers.Builder, recordbatches::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, recordbatches, 0) footerStartRecordBatchesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 24, numelems, 8) footerEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Block <: FlatBuffers.Struct bytes::Vector{UInt8} pos::Base.Int end FlatBuffers.structsizeof(::Base.Type{Block}) = 24 Base.propertynames(x::Block) = (:offset, :metaDataLength, :bodyLength) function Base.getproperty(x::Block, field::Symbol) if field === :offset return FlatBuffers.get(x, FlatBuffers.pos(x), Int64) elseif field === :metaDataLength return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int32) elseif field === :bodyLength return FlatBuffers.get(x, FlatBuffers.pos(x) + 16, Int64) end return nothing end function createBlock( b::FlatBuffers.Builder, offset::Int64, metadatalength::Int32, bodylength::Int64, ) FlatBuffers.prep!(b, 8, 24) prepend!(b, bodylength) FlatBuffers.pad!(b, 4) prepend!(b, metadatalength) prepend!(b, offset) return FlatBuffers.offset(b) end ================================================ FILE: src/metadata/Flatbuf.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. module Flatbuf using EnumX using ..FlatBuffers include("Schema.jl") include("File.jl") include("Message.jl") end # module ================================================ FILE: src/metadata/Message.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct FieldNode <: FlatBuffers.Struct bytes::Vector{UInt8} pos::Base.Int end FlatBuffers.structsizeof(::Base.Type{FieldNode}) = 16 Base.propertynames(x::FieldNode) = (:length, :null_count) function Base.getproperty(x::FieldNode, field::Symbol) if field === :length return FlatBuffers.get(x, FlatBuffers.pos(x), Int64) elseif field === :null_count return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int64) end return nothing end function createFieldNode(b::FlatBuffers.Builder, length::Int64, nullCount::Int64) FlatBuffers.prep!(b, 8, 16) prepend!(b, nullCount) prepend!(b, length) return FlatBuffers.offset(b) end @enumx CompressionType::Int8 LZ4_FRAME ZSTD @enumx BodyCompressionMethod::Int8 BUFFER struct BodyCompression <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::BodyCompression) = (:codec, :method) function Base.getproperty(x::BodyCompression, field::Symbol) if field === :codec o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), CompressionType.T) return CompressionType.LZ4_FRAME elseif field === :method o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), BodyCompressionMethod.T) return BodyCompressionMethod.BUFFER end return nothing end bodyCompressionStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) bodyCompressionAddCodec(b::FlatBuffers.Builder, codec::CompressionType.T) = FlatBuffers.prependslot!(b, 0, codec, 0) bodyCompressionAddMethod(b::FlatBuffers.Builder, method::BodyCompressionMethod.T) = FlatBuffers.prependslot!(b, 1, method, 0) bodyCompressionEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct RecordBatch <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::RecordBatch) = (:length, :nodes, :buffers, :compression, :variadicBufferCounts) function Base.getproperty(x::RecordBatch, field::Symbol) if field === :length o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64) elseif field === :nodes o = FlatBuffers.offset(x, 6) if o != 0 return FlatBuffers.Array{FieldNode}(x, o) end elseif field === :buffers o = FlatBuffers.offset(x, 8) if o != 0 return FlatBuffers.Array{Buffer}(x, o) end elseif field === :compression o = FlatBuffers.offset(x, 10) if o != 0 y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x)) return FlatBuffers.init(BodyCompression, FlatBuffers.bytes(x), y) end elseif field === :variadicBufferCounts o = FlatBuffers.offset(x, 12) if o != 0 return FlatBuffers.Array{Int32}(x, o) end end return nothing end recordBatchStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 4) recordBatchAddLength(b::FlatBuffers.Builder, length::Int64) = FlatBuffers.prependslot!(b, 0, length, 0) recordBatchAddNodes(b::FlatBuffers.Builder, nodes::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, nodes, 0) recordBatchStartNodesVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 16, numelems, 8) recordBatchAddBuffers(b::FlatBuffers.Builder, buffers::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, buffers, 0) recordBatchStartBuffersVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 16, numelems, 8) recordBatchAddCompression(b::FlatBuffers.Builder, c::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, c, 0) recordBatchEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct DictionaryBatch <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::DictionaryBatch) = (:id, :data, :isDelta) function Base.getproperty(x::DictionaryBatch, field::Symbol) if field === :id o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64) return Int64(0) elseif field === :data o = FlatBuffers.offset(x, 6) if o != 0 y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x)) return FlatBuffers.init(RecordBatch, FlatBuffers.bytes(x), y) end elseif field === :isDelta o = FlatBuffers.offset(x, 8) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool) return false end return nothing end dictionaryBatchStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3) dictionaryBatchAddId(b::FlatBuffers.Builder, id::Int64) = FlatBuffers.prependslot!(b, 0, id, 0) dictionaryBatchAddData(b::FlatBuffers.Builder, data::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, data, 0) dictionaryBatchAddIsDelta(b::FlatBuffers.Builder, isdelta::Base.Bool) = FlatBuffers.prependslot!(b, 2, isdelta, false) dictionaryBatchEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) function MessageHeader(b::UInt8) b == 1 && return Schema b == 2 && return DictionaryBatch b == 3 && return RecordBatch # b == 4 && return Tensor # b == 5 && return SparseTensor return nothing end function MessageHeader(::Base.Type{T})::Int16 where {T} T == Schema && return 1 T == DictionaryBatch && return 2 T == RecordBatch && return 3 # T == Tensor && return 4 # T == SparseTensor && return 5 return 0 end struct Message <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Message) = (:version, :header, :bodyLength, :custom_metadata) function Base.getproperty(x::Message, field::Symbol) if field === :version o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), MetadataVersion.T) elseif field === :header o = FlatBuffers.offset(x, 6) if o != 0 T = MessageHeader(FlatBuffers.get(x, o + FlatBuffers.pos(x), UInt8)) o = FlatBuffers.offset(x, 8) pos = FlatBuffers.union(x, o) if o != 0 return FlatBuffers.init(T, FlatBuffers.bytes(x), pos) end end elseif field === :bodyLength o = FlatBuffers.offset(x, 10) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64) return Int64(0) elseif field === :custom_metadata o = FlatBuffers.offset(x, 12) if o != 0 return FlatBuffers.Array{KeyValue}(x, o) end end return nothing end messageStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 5) messageAddVersion(b::FlatBuffers.Builder, version::MetadataVersion.T) = FlatBuffers.prependslot!(b, 0, version, 0) messageAddHeaderType(b::FlatBuffers.Builder, ::Core.Type{T}) where {T} = FlatBuffers.prependslot!(b, 1, MessageHeader(T), 0) messageAddHeader(b::FlatBuffers.Builder, header::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, header, 0) messageAddBodyLength(b::FlatBuffers.Builder, bodyLength::Int64) = FlatBuffers.prependslot!(b, 3, bodyLength, 0) messageAddCustomMetadata(b::FlatBuffers.Builder, meta::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 4, meta, 0) messageStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) messageEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) ================================================ FILE: src/metadata/Schema.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @enumx MetadataVersion::Int16 V1 V2 V3 V4 V5 struct Null <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Null) = () nullStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) nullEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Struct <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Struct) = () structStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) structEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct List <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::List) = () listStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) listEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct LargeList <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::LargeList) = () largeListStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) largeListEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct FixedSizeList <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::FixedSizeList) = (:listSize,) function Base.getproperty(x::FixedSizeList, field::Symbol) if field === :listSize o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) return Int32(0) end return nothing end fixedSizeListStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) fixedSizeListAddListSize(b::FlatBuffers.Builder, listSize::Int32) = FlatBuffers.prependslot!(b, 0, listSize, 0) fixedSizeListEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Map <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Map) = (:keysSorted,) function Base.getproperty(x::Map, field::Symbol) if field === :keysSorted o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool) end return nothing end mapStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) mapAddKeysSorted(b::FlatBuffers.Builder, keyssorted::Base.Bool) = FlatBuffers.prependslot!(b, 0, keyssorted, 0) mapEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx UnionMode::Int16 Sparse Dense struct Union <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Union) = (:mode, :typeIds) function Base.getproperty(x::Union, field::Symbol) if field === :mode o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), UnionMode.T) return UnionMode.Sparse elseif field === :typeIds o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.Array{Int32}(x, o) end return nothing end unionStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) unionAddMode(b::FlatBuffers.Builder, mode::UnionMode.T) = FlatBuffers.prependslot!(b, 0, mode, 0) unionAddTypeIds(b::FlatBuffers.Builder, typeIds::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, typeIds, 0) unionStartTypeIdsVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) unionEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Int <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Int) = (:bitWidth, :is_signed) function Base.getproperty(x::Int, field::Symbol) if field === :bitWidth o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) elseif field === :is_signed o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool) return false end return nothing end intStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) intAddBitWidth(b::FlatBuffers.Builder, bitwidth::Int32) = FlatBuffers.prependslot!(b, 0, bitwidth, 0) intAddIsSigned(b::FlatBuffers.Builder, issigned::Base.Bool) = FlatBuffers.prependslot!(b, 1, issigned, 0) intEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx Precision::Int16 HALF SINGLE DOUBLE struct FloatingPoint <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::FloatingPoint) = (:precision,) function Base.getproperty(x::FloatingPoint, field::Symbol) if field === :precision o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Precision.T) return Precision.HALF end return nothing end floatingPointStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) floatingPointAddPrecision(b::FlatBuffers.Builder, precision::Precision.T) = FlatBuffers.prependslot!(b, 0, precision, 0) floatingPointEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Utf8 <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Utf8) = () utf8Start(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) utf8End(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Binary <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Binary) = () binaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) binaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct LargeUtf8 <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::LargeUtf8) = () largUtf8Start(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) largUtf8End(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct LargeBinary <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::LargeBinary) = () largeBinaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) largeBinaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct FixedSizeBinary <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::FixedSizeBinary) = (:byteWidth,) function Base.getproperty(x::FixedSizeBinary, field::Symbol) if field === :byteWidth o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) end return nothing end fixedSizeBinaryStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) fixedSizeBinaryAddByteWidth(b::FlatBuffers.Builder, bytewidth::Int32) = FlatBuffers.prependslot!(b, 0, bytewidth, 0) fixedSizeBinaryEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Bool <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Bool) = () boolStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) boolEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Decimal <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Decimal) = (:precision, :scale, :bitWidth) function Base.getproperty(x::Decimal, field::Symbol) if field === :precision o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) return Int32(0) elseif field === :scale o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) return Int32(0) elseif field === :bitWidth o = FlatBuffers.offset(x, 8) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) return Int32(128) end return nothing end decimalStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3) decimalAddPrecision(b::FlatBuffers.Builder, precision::Int32) = FlatBuffers.prependslot!(b, 0, precision, 0) decimalAddScale(b::FlatBuffers.Builder, scale::Int32) = FlatBuffers.prependslot!(b, 1, scale, 0) decimalAddBitWidth(b::FlatBuffers.Builder, bitWidth::Int32) = FlatBuffers.prependslot!(b, 2, bitWidth, Int32(128)) decimalEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx DateUnit::Int16 DAY MILLISECOND struct Date <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Date) = (:unit,) function Base.getproperty(x::Date, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DateUnit.T) return DateUnit.MILLISECOND end return nothing end dateStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) dateAddUnit(b::FlatBuffers.Builder, unit::DateUnit.T) = FlatBuffers.prependslot!(b, 0, unit, 1) dateEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx TimeUnit::Int16 SECOND MILLISECOND MICROSECOND NANOSECOND struct Time <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Time) = (:unit, :bitWidth) function Base.getproperty(x::Time, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit.T) return TimeUnit.MILLISECOND elseif field === :bitWidth o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int32) return 32 end return nothing end timeStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) timeAddUnit(b::FlatBuffers.Builder, unit::TimeUnit.T) = FlatBuffers.prependslot!(b, 0, unit, 1) timeAddBitWidth(b::FlatBuffers.Builder, bitwidth::Int32) = FlatBuffers.prependslot!(b, 1, bitwidth, 32) timeEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Timestamp <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Timestamp) = (:unit, :timezone) function Base.getproperty(x::Timestamp, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit.T) return TimeUnit.SECOND elseif field === :timezone o = FlatBuffers.offset(x, 6) o != 0 && return String(x, o + FlatBuffers.pos(x)) end return nothing end timestampStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) timestampAddUnit(b::FlatBuffers.Builder, unit::TimeUnit.T) = FlatBuffers.prependslot!(b, 0, unit, 0) timestampAddTimezone(b::FlatBuffers.Builder, timezone::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, timezone, 0) timestampEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx IntervalUnit::Int16 YEAR_MONTH DAY_TIME struct Interval <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Interval) = (:unit,) function Base.getproperty(x::Interval, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), IntervalUnit.T) return IntervalUnit.YEAR_MONTH end return nothing end intervalStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) intervalAddUnit(b::FlatBuffers.Builder, unit::IntervalUnit.T) = FlatBuffers.prependslot!(b, 0, unit, 0) intervalEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Duration <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Duration) = (:unit,) function Base.getproperty(x::Duration, field::Symbol) if field === :unit o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), TimeUnit.T) return TimeUnit.MILLISECOND end return nothing end durationStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 1) durationAddUnit(b::FlatBuffers.Builder, unit::TimeUnit.T) = FlatBuffers.prependslot!(b, 0, unit, 1) durationEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # /// Contains two child arrays, run_ends and values. # /// The run_ends child array must be a 16/32/64-bit integer array # /// which encodes the indices at which the run with the value in # /// each corresponding index in the values child array ends. # /// Like list/struct types, the value array can be of any type. # table RunEndEncoded { # } struct RunEndEncoded <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::RunEndEncoded) = () runEndEncodedStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) runEndEncodedEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # /// Logically the same as Binary, but the internal representation uses a view # /// struct that contains the string length and either the string's entire data # /// inline (for small strings) or an inlined prefix, an index of another buffer, # /// and an offset pointing to a slice in that buffer (for non-small strings). # /// # /// Since it uses a variable number of data buffers, each Field with this type # /// must have a corresponding entry in `variadicBufferCounts`. # table BinaryView { # } struct BinaryView <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::BinaryView) = () binaryViewStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) binaryViewEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # /// Logically the same as Utf8, but the internal representation uses a view # /// struct that contains the string length and either the string's entire data # /// inline (for small strings) or an inlined prefix, an index of another buffer, # /// and an offset pointing to a slice in that buffer (for non-small strings). # /// # /// Since it uses a variable number of data buffers, each Field with this type # /// must have a corresponding entry in `variadicBufferCounts`. # table Utf8View { # } struct Utf8View <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Utf8View) = () utf8ViewStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) utf8ViewEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # /// Represents the same logical types that List can, but contains offsets and # /// sizes allowing for writes in any order and sharing of child values among # /// list values. # table ListView { # } struct ListView <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::ListView) = () listViewStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) listViewEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # /// Represents the same logical types that LargeList can, but contains offsets # /// and sizes allowing for writes in any order and sharing of child values among # /// list values. # table LargeListView { # } struct LargeListView <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::LargeListView) = () largeListViewStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 0) largeListViewEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) # Ensure this binding is distinct from `Base.Type` by forward-declaring it. Otherwise, in # Julia 1.12, defining a function called `Type` by simply defining methods ends up actually # defining methods for `Base.Type`. function Type end function Type(b::UInt8) b == 1 && return Null b == 2 && return Int b == 3 && return FloatingPoint b == 4 && return Binary b == 5 && return Utf8 b == 6 && return Bool b == 7 && return Decimal b == 8 && return Date b == 9 && return Time b == 10 && return Timestamp b == 11 && return Interval b == 12 && return List b == 13 && return Struct b == 14 && return Union b == 15 && return FixedSizeBinary b == 16 && return FixedSizeList b == 17 && return Map b == 18 && return Duration b == 19 && return LargeBinary b == 20 && return LargeUtf8 b == 21 && return LargeList b == 22 && return RunEndEncoded b == 23 && return BinaryView b == 24 && return Utf8View b == 25 && return ListView b == 26 && return LargeListView return nothing end function Type(::Base.Type{T})::Int16 where {T} T == Null && return 1 T == Int && return 2 T == FloatingPoint && return 3 T == Binary && return 4 T == Utf8 && return 5 T == Bool && return 6 T == Decimal && return 7 T == Date && return 8 T == Time && return 9 T == Timestamp && return 10 T == Interval && return 11 T == List && return 12 T == Struct && return 13 T == Union && return 14 T == FixedSizeBinary && return 15 T == FixedSizeList && return 16 T == Map && return 17 T == Duration && return 18 T == LargeBinary && return 19 T == LargeUtf8 && return 20 T == LargeList && return 21 return 0 end struct KeyValue <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::KeyValue) = (:key, :value) function Base.getproperty(x::KeyValue, field::Symbol) if field === :key o = FlatBuffers.offset(x, 4) o != 0 && return String(x, o + FlatBuffers.pos(x)) elseif field === :value o = FlatBuffers.offset(x, 6) o != 0 && return String(x, o + FlatBuffers.pos(x)) end return nothing end keyValueStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 2) keyValueAddKey(b::FlatBuffers.Builder, key::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 0, key, 0) keyValueAddValue(b::FlatBuffers.Builder, value::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, value, 0) keyValueEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx DictionaryKind::Int16 DenseArray struct DictionaryEncoding <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::DictionaryEncoding) = (:id, :indexType, :isOrdered, :dictionaryKind) function Base.getproperty(x::DictionaryEncoding, field::Symbol) if field === :id o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Int64) return Int64(0) elseif field === :indexType o = FlatBuffers.offset(x, 6) if o != 0 y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x)) return FlatBuffers.init(Int, FlatBuffers.bytes(x), y) end elseif field === :isOrdered o = FlatBuffers.offset(x, 8) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool) return false elseif field === :dictionaryKind o = FlatBuffers.offset(x, 10) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), DictionaryKind.T) end return nothing end dictionaryEncodingStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3) dictionaryEncodingAddId(b::FlatBuffers.Builder, id::Int64) = FlatBuffers.prependslot!(b, 0, id, 0) dictionaryEncodingAddIndexType(b::FlatBuffers.Builder, indextype::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, indextype, 0) dictionaryEncodingAddIsOrdered(b::FlatBuffers.Builder, isordered::Base.Bool) = FlatBuffers.prependslot!(b, 1, isordered, 0) dictionaryEncodingEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) struct Field <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Field) = (:name, :nullable, :type, :dictionary, :children, :custom_metadata) function Base.getproperty(x::Field, field::Symbol) if field === :name o = FlatBuffers.offset(x, 4) o != 0 && return String(x, o + FlatBuffers.pos(x)) elseif field === :nullable o = FlatBuffers.offset(x, 6) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Base.Bool) return false elseif field === :type o = FlatBuffers.offset(x, 8) if o != 0 T = Type(FlatBuffers.get(x, o + FlatBuffers.pos(x), UInt8)) o = FlatBuffers.offset(x, 10) pos = FlatBuffers.union(x, o) if o != 0 return FlatBuffers.init(T, FlatBuffers.bytes(x), pos) end end elseif field === :dictionary o = FlatBuffers.offset(x, 12) if o != 0 y = FlatBuffers.indirect(x, o + FlatBuffers.pos(x)) return FlatBuffers.init(DictionaryEncoding, FlatBuffers.bytes(x), y) end elseif field === :children o = FlatBuffers.offset(x, 14) if o != 0 return FlatBuffers.Array{Field}(x, o) end elseif field === :custom_metadata o = FlatBuffers.offset(x, 16) if o != 0 return FlatBuffers.Array{KeyValue}(x, o) end end return nothing end fieldStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 7) fieldAddName(b::FlatBuffers.Builder, name::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 0, name, 0) fieldAddNullable(b::FlatBuffers.Builder, nullable::Base.Bool) = FlatBuffers.prependslot!(b, 1, nullable, false) fieldAddTypeType(b::FlatBuffers.Builder, ::Core.Type{T}) where {T} = FlatBuffers.prependslot!(b, 2, Type(T), 0) fieldAddType(b::FlatBuffers.Builder, type::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 3, type, 0) fieldAddDictionary(b::FlatBuffers.Builder, dictionary::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 4, dictionary, 0) fieldAddChildren(b::FlatBuffers.Builder, children::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 5, children, 0) fieldStartChildrenVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) fieldAddCustomMetadata(b::FlatBuffers.Builder, custommetadata::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 6, custommetadata, 0) fieldStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) fieldEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) @enumx Endianness::Int16 Little Big struct Buffer <: FlatBuffers.Struct bytes::Vector{UInt8} pos::Base.Int end FlatBuffers.structsizeof(::Base.Type{Buffer}) = 16 Base.propertynames(x::Buffer) = (:offset, :length) function Base.getproperty(x::Buffer, field::Symbol) if field === :offset return FlatBuffers.get(x, FlatBuffers.pos(x), Int64) elseif field === :length return FlatBuffers.get(x, FlatBuffers.pos(x) + 8, Int64) end return nothing end function createBuffer(b::FlatBuffers.Builder, offset::Int64, length::Int64) FlatBuffers.prep!(b, 8, 16) prepend!(b, length) prepend!(b, offset) return FlatBuffers.offset(b) end struct Schema <: FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end Base.propertynames(x::Schema) = (:endianness, :fields, :custom_metadata) function Base.getproperty(x::Schema, field::Symbol) if field === :endianness o = FlatBuffers.offset(x, 4) o != 0 && return FlatBuffers.get(x, o + FlatBuffers.pos(x), Endianness.T) elseif field === :fields o = FlatBuffers.offset(x, 6) if o != 0 return FlatBuffers.Array{Field}(x, o) end elseif field === :custom_metadata o = FlatBuffers.offset(x, 8) if o != 0 return FlatBuffers.Array{KeyValue}(x, o) end end return nothing end schemaStart(b::FlatBuffers.Builder) = FlatBuffers.startobject!(b, 3) schemaAddEndianness(b::FlatBuffers.Builder, endianness::Endianness.T) = FlatBuffers.prependslot!(b, 0, endianness, 0) schemaAddFields(b::FlatBuffers.Builder, fields::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 1, fields, 0) schemaStartFieldsVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) schemaAddCustomMetadata(b::FlatBuffers.Builder, custommetadata::FlatBuffers.UOffsetT) = FlatBuffers.prependoffsetslot!(b, 2, custommetadata, 0) schemaStartCustomMetadataVector(b::FlatBuffers.Builder, numelems) = FlatBuffers.startvector!(b, 4, numelems, 4) schemaEnd(b::FlatBuffers.Builder) = FlatBuffers.endobject!(b) ================================================ FILE: src/show.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # 2-arg show: show schema and list # of metadata entries if non-zero function Base.show(io::IO, table::Table) ncols = length(Tables.columnnames(table)) print(io, "$(typeof(table)) with $(Tables.rowcount(table)) rows, $(ncols) columns,") meta = getmetadata(table) if meta !== nothing && !isempty(meta) print(io, " ", length(meta), " metadata entries,") end sch = Tables.schema(table) print(io, " and schema:\n") show(IOContext(io, :print_schema_header => false), sch) return nothing end # 3-arg show: show schema and show metadata entries adaptively according to `displaysize` function Base.show(io::IO, mime::MIME"text/plain", table::Table) display_rows, display_cols = displaysize(io) ncols = length(Tables.columnnames(table)) meta = getmetadata(table) if meta !== nothing display_rows -= 1 # decrement for metadata header line display_rows -= min(length(meta), 2) # decrement so we can show at least 2 lines of metadata end print( io, "$(typeof(table)) with $(Tables.rowcount(table)) rows, $(ncols) columns, and ", ) sch = Tables.schema(table) print(io, "schema:\n") schema_context = IOContext( io, :print_schema_header => false, :displaysize => (max(display_rows, 3), display_cols), ) schema_str = sprint(show, mime, sch; context=schema_context) print(io, schema_str) display_rows -= (count("\n", schema_str) + 1) # decrement for number of lines printed if meta !== nothing print(io, "\n\nwith metadata given by a ") show( IOContext(io, :displaysize => (max(display_rows, 5), display_cols)), mime, meta, ) end return nothing end ================================================ FILE: src/table.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. struct ArrowBlob bytes::Vector{UInt8} pos::Int len::Int end ArrowBlob(bytes::Vector{UInt8}, pos::Int, len::Nothing) = ArrowBlob(bytes, pos, length(bytes)) tobytes(bytes::Vector{UInt8}) = bytes tobytes(io::IO) = Base.read(io) tobytes(io::IOStream) = Mmap.mmap(io) tobytes(file_path) = open(tobytes, file_path, "r") struct BatchIterator bytes::Vector{UInt8} startpos::Int function BatchIterator(blob::ArrowBlob) bytes, pos, len = blob.bytes, blob.pos, blob.len if len > 24 && _startswith(bytes, pos, FILE_FORMAT_MAGIC_BYTES) pos += 8 # skip past magic bytes + padding end new(bytes, pos) end end """ Arrow.Stream(io::IO; convert::Bool=true) Arrow.Stream(file::String; convert::Bool=true) Arrow.Stream(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true) Arrow.Stream(inputs::Vector; convert::Bool=true) Start reading an arrow formatted table, from: * `io`, bytes will be read all at once via `read(io)` * `file`, bytes will be read via `Mmap.mmap(file)` * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len` * A `Vector` of any of the above, in which each input should be an IPC or arrow file and must match schema Reads the initial schema message from the arrow stream/file, then returns an `Arrow.Stream` object which will iterate over record batch messages, producing an [`Arrow.Table`](@ref) on each iteration. By iterating [`Arrow.Table`](@ref), `Arrow.Stream` satisfies the `Tables.partitions` interface, and as such can be passed to Tables.jl-compatible sink functions. This allows iterating over extremely large "arrow tables" in chunks represented as record batches. Supports the `convert` keyword argument which controls whether certain arrow primitive types will be lazily converted to more friendly Julia defaults; by default, `convert=true`. """ mutable struct Stream inputs::Vector{ArrowBlob} inputindex::Int batchiterator::Union{Nothing,BatchIterator} names::Vector{Symbol} types::Vector{Type} schema::Union{Nothing,Meta.Schema} dictencodings::Lockable{Dict{Int64,DictEncoding}} # dictionary id => DictEncoding dictencoded::Dict{Int64,Meta.Field} # dictionary id => field convert::Bool compression::Ref{Union{Symbol,Nothing}} end function Stream(inputs::Vector{ArrowBlob}; convert::Bool=true) inputindex = 1 batchiterator = nothing names = Symbol[] types = Type[] schema = nothing dictencodings = Lockable(Dict{Int64,DictEncoding}()) dictencoded = Dict{Int64,Meta.Field}() compression = Ref{Union{Symbol,Nothing}}(nothing) Stream( inputs, inputindex, batchiterator, names, types, schema, dictencodings, dictencoded, convert, compression, ) end function Stream(input, pos::Integer=1, len=nothing; kw...) b = tobytes(input) isempty(b) ? Stream(ArrowBlob[]; kw...) : Stream([ArrowBlob(b, pos, len)]; kw...) end function Stream(input::Vector{UInt8}, pos::Integer=1, len=nothing; kw...) b = tobytes(input) isempty(b) ? Stream(ArrowBlob[]; kw...) : Stream([ArrowBlob(b, pos, len)]; kw...) end function Stream(inputs::AbstractVector; kw...) blobs = ArrowBlob[] for x in inputs b = tobytes(x) isempty(b) && continue push!(blobs, ArrowBlob(b, 1, nothing)) end Stream(blobs; kw...) end function initialize!(x::Stream) isempty(getfield(x, :names)) || return # Initialize member fields using iteration and reset state lastinputindex = x.inputindex lastbatchiterator = x.batchiterator iterate(x) x.inputindex = lastinputindex x.batchiterator = lastbatchiterator nothing end Tables.partitions(x::Stream) = x function Tables.columnnames(x::Stream) initialize!(x) getfield(x, :names) end function Tables.schema(x::Stream) initialize!(x) Tables.Schema(Tables.columnnames(x), getfield(x, :types)) end Base.IteratorSize(::Type{Stream}) = Base.SizeUnknown() Base.eltype(::Type{Stream}) = Table Base.isdone(x::Stream) = x.inputindex > length(x.inputs) function Base.iterate(x::Stream, (pos, id)=(1, 0)) if Base.isdone(x) x.inputindex = 1 x.batchiterator = nothing return nothing end if isnothing(x.batchiterator) blob = x.inputs[x.inputindex] x.batchiterator = BatchIterator(blob) pos = x.batchiterator.startpos end columns = AbstractVector[] compression = nothing while true state = iterate(x.batchiterator, (pos, id)) # check for additional inputs while state === nothing x.inputindex += 1 if Base.isdone(x) x.inputindex = 1 x.batchiterator = nothing return nothing end blob = x.inputs[x.inputindex] x.batchiterator = BatchIterator(blob) pos = x.batchiterator.startpos state = iterate(x.batchiterator, (pos, id)) end batch, (pos, id) = state header = batch.msg.header if isnothing(x.schema) && !isa(header, Meta.Schema) throw(ArgumentError("first arrow ipc message MUST be a schema message")) end if header isa Meta.Schema if isnothing(x.schema) x.schema = header # assert endianness? # store custom_metadata? for (i, field) in enumerate(x.schema.fields) push!(x.names, Symbol(field.name)) push!( x.types, juliaeltype(field, buildmetadata(field.custom_metadata), x.convert), ) # recursively find any dictionaries for any fields getdictionaries!(x.dictencoded, field) @debug "parsed column from schema: field = $field" end elseif header != x.schema throw( ArgumentError( "mismatched schemas between different arrow batches: $(x.schema) != $header", ), ) end elseif header isa Meta.DictionaryBatch id = header.id recordbatch = header.data @debug "parsing dictionary batch message: id = $id, compression = $(recordbatch.compression)" if recordbatch.compression !== nothing compression = recordbatch.compression end @lock x.dictencodings begin dictencodings = x.dictencodings[] if haskey(dictencodings, id) && header.isDelta # delta field = x.dictencoded[id] values, _, _, _ = build( field, field.type, batch, recordbatch, x.dictencodings, Int64(1), Int64(1), Int64(1), x.convert, ) dictencoding = dictencodings[id] append!(dictencoding.data, values) continue end # new dictencoding or replace field = x.dictencoded[id] values, _, _, _ = build( field, field.type, batch, recordbatch, x.dictencodings, Int64(1), Int64(1), Int64(1), x.convert, ) A = ChainedVector([values]) S = field.dictionary.indexType === nothing ? Int32 : juliaeltype(field, field.dictionary.indexType, false) dictencodings[id] = DictEncoding{eltype(A),S,typeof(A)}( id, A, field.dictionary.isOrdered, values.metadata, ) end # lock @debug "parsed dictionary batch message: id=$id, data=$values\n" elseif header isa Meta.RecordBatch @debug "parsing record batch message: compression = $(header.compression)" if header.compression !== nothing compression = header.compression end for vec in VectorIterator(x.schema, batch, x.dictencodings, x.convert) push!(columns, vec) end break else throw(ArgumentError("unsupported arrow message type: $(typeof(header))")) end end if compression !== nothing if compression.codec == Flatbuf.CompressionType.ZSTD x.compression[] = :zstd elseif compression.codec == Flatbuf.CompressionType.LZ4_FRAME x.compression[] = :lz4 else throw(ArgumentError("unsupported compression codec: $(compression.codec)")) end end lookup = Dict{Symbol,AbstractVector}() types = Type[] for (nm, col) in zip(x.names, columns) lookup[nm] = col push!(types, eltype(col)) end return Table(x.names, types, columns, lookup, Ref(x.schema)), (pos, id) end """ Arrow.Table(io::IO; convert::Bool=true) Arrow.Table(file::String; convert::Bool=true) Arrow.Table(bytes::Vector{UInt8}, pos=1, len=nothing; convert::Bool=true) Arrow.Table(inputs::Vector; convert::Bool=true) Read an arrow formatted table, from: * `io`, bytes will be read all at once via `read(io)` * `file`, bytes will be read via `Mmap.mmap(file)` * `bytes`, a byte vector directly, optionally allowing specifying the starting byte position `pos` and `len` * A `Vector` of any of the above, in which each input should be an IPC or arrow file and must match schema Returns a `Arrow.Table` object that allows column access via `table.col1`, `table[:col1]`, or `table[1]`. NOTE: the columns in an `Arrow.Table` are views into the original arrow memory, and hence are not easily modifiable (with e.g. `push!`, `append!`, etc.). To mutate arrow columns, call `copy(x)` to materialize the arrow data as a normal Julia array. `Arrow.Table` also satisfies the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface, and so can easily be materialied via any supporting sink function: e.g. `DataFrame(Arrow.Table(file))`, `SQLite.load!(db, "table", Arrow.Table(file))`, etc. Supports the `convert` keyword argument which controls whether certain arrow primitive types will be lazily converted to more friendly Julia defaults; by default, `convert=true`. """ struct Table <: Tables.AbstractColumns names::Vector{Symbol} types::Vector{Type} columns::Vector{AbstractVector} lookup::Dict{Symbol,AbstractVector} schema::Ref{Meta.Schema} metadata::Ref{Union{Nothing,Base.ImmutableDict{String,String}}} end Table() = Table( Symbol[], Type[], AbstractVector[], Dict{Symbol,AbstractVector}(), Ref{Meta.Schema}(), Ref{Union{Nothing,Base.ImmutableDict{String,String}}}(nothing), ) function Table(names, types, columns, lookup, schema) m = isassigned(schema) ? buildmetadata(schema[]) : nothing return Table( names, types, columns, lookup, schema, Ref{Union{Nothing,Base.ImmutableDict{String,String}}}(m), ) end names(t::Table) = getfield(t, :names) types(t::Table) = getfield(t, :types) columns(t::Table) = getfield(t, :columns) lookup(t::Table) = getfield(t, :lookup) schema(t::Table) = getfield(t, :schema) metadata(t::Table) = getfield(t, :metadata) """ Arrow.getmetadata(x) If `x isa Arrow.Table` return a `Base.ImmutableDict{String,String}` representation of `x`'s `Schema` `custom_metadata`, or `nothing` if no such metadata exists. If `x isa Arrow.ArrowVector`, return a `Base.ImmutableDict{String,String}` representation of `x`'s `Field` `custom_metadata`, or `nothing` if no such metadata exists. Otherwise, return `nothing`. See [the official Arrow documentation for more details on custom application metadata](https://arrow.apache.org/docs/format/Columnar.html#custom-application-metadata). """ getmetadata(t::Table) = getfield(t, :metadata)[] getmetadata(::Any) = nothing DataAPI.metadatasupport(::Type{Table}) = (read=true, write=false) DataAPI.colmetadatasupport(::Type{Table}) = (read=true, write=false) function DataAPI.metadata(t::Table, key::AbstractString; style::Bool=false) meta = getmetadata(t)[key] return style ? (meta, :default) : meta end function DataAPI.metadata(t::Table, key::AbstractString, default; style::Bool=false) meta = getmetadata(t) if meta !== nothing haskey(meta, key) && return style ? meta[key] : (meta[key], :default) end return style ? (default, :default) : default end function DataAPI.metadatakeys(t::Table) meta = getmetadata(t) meta === nothing && return () return keys(meta) end function DataAPI.colmetadata(t::Table, col, key::AbstractString; style::Bool=false) meta = getmetadata(t[col])[key] return style ? (meta, :default) : meta end function DataAPI.colmetadata(t::Table, col, key::AbstractString, default; style::Bool=false) meta = getmetadata(t[col]) if meta !== nothing haskey(meta, key) && return style ? (meta[key], :default) : meta[key] end return style ? (default, :default) : default end function DataAPI.colmetadatakeys(t::Table, col) meta = getmetadata(t[col]) meta === nothing && return () return keys(meta) end function DataAPI.colmetadatakeys(t::Table) return ( col => DataAPI.colmetadatakeys(t, col) for col in Tables.columnnames(t) if getmetadata(t[col]) !== nothing ) end Tables.istable(::Table) = true Tables.columnaccess(::Table) = true Tables.columns(t::Table) = Tables.CopiedColumns(t) Tables.schema(t::Table) = Tables.Schema(names(t), types(t)) Tables.columnnames(t::Table) = names(t) Tables.getcolumn(t::Table, i::Int) = columns(t)[i] Tables.getcolumn(t::Table, nm::Symbol) = lookup(t)[nm] struct TablePartitions table::Table npartitions::Int end function TablePartitions(table::Table) cols = columns(table) npartitions = if length(cols) == 0 0 elseif cols[1] isa ChainedVector length(cols[1].arrays) else 1 end return TablePartitions(table, npartitions) end function Base.iterate(tp::TablePartitions, i=1) i > tp.npartitions && return nothing tp.npartitions == 1 && return tp.table, i + 1 cols = columns(tp.table) newcols = AbstractVector[cols[j].arrays[i] for j = 1:length(cols)] nms = names(tp.table) tbl = Table( nms, types(tp.table), newcols, Dict{Symbol,AbstractVector}(nms[i] => newcols[i] for i = 1:length(nms)), schema(tp.table), ) return tbl, i + 1 end Tables.partitions(t::Table) = TablePartitions(t) # high-level user API functions Table(input, pos::Integer=1, len=nothing; kw...) = Table([ArrowBlob(tobytes(input), pos, len)]; kw...) Table(input::Vector{UInt8}, pos::Integer=1, len=nothing; kw...) = Table([ArrowBlob(tobytes(input), pos, len)]; kw...) Table(inputs::Vector; kw...) = Table([ArrowBlob(tobytes(x), 1, nothing) for x in inputs]; kw...) # will detect whether we're reading a Table from a file or stream function Table(blobs::Vector{ArrowBlob}; convert::Bool=true) t = Table() sch = nothing dictencodingslockable = Lockable(Dict{Int64,DictEncoding}()) # dictionary id => DictEncoding dictencoded = Dict{Int64,Meta.Field}() # dictionary id => field # we'll grow/add a record batch set of columns as they're constructed # must be holding the lock while growing/adding # starts at 0-length because we don't know how many record batches there will be rb_cols = [] rb_cols_lock = ReentrantLock() rbi = 1 tasks = Task[] for blob in blobs for batch in BatchIterator(blob) # store custom_metadata of batch.msg? header = batch.msg.header if header isa Meta.Schema @debug "parsing schema message" # assert endianness? # store custom_metadata? if sch === nothing for (i, field) in enumerate(header.fields) push!(names(t), Symbol(field.name)) # recursively find any dictionaries for any fields getdictionaries!(dictencoded, field) @debug "parsed column from schema: field = $field" end sch = header schema(t)[] = sch elseif sch != header throw( ArgumentError( "mismatched schemas between different arrow batches: $sch != $header", ), ) end elseif header isa Meta.DictionaryBatch id = header.id recordbatch = header.data @debug "parsing dictionary batch message: id = $id, compression = $(recordbatch.compression)" @lock dictencodingslockable begin dictencodings = dictencodingslockable[] if haskey(dictencodings, id) && header.isDelta # delta field = dictencoded[id] values, _, _, _ = build( field, field.type, batch, recordbatch, dictencodingslockable, Int64(1), Int64(1), Int64(1), convert, ) dictencoding = dictencodings[id] if typeof(dictencoding.data) <: ChainedVector append!(dictencoding.data, values) else A = ChainedVector([dictencoding.data, values]) S = field.dictionary.indexType === nothing ? Int32 : juliaeltype(field, field.dictionary.indexType, false) dictencodings[id] = DictEncoding{eltype(A),S,typeof(A)}( id, A, field.dictionary.isOrdered, values.metadata, ) end continue end # new dictencoding or replace field = dictencoded[id] values, _, _, _ = build( field, field.type, batch, recordbatch, dictencodingslockable, Int64(1), Int64(1), Int64(1), convert, ) A = values S = field.dictionary.indexType === nothing ? Int32 : juliaeltype(field, field.dictionary.indexType, false) dictencodings[id] = DictEncoding{eltype(A),S,typeof(A)}( id, A, field.dictionary.isOrdered, values.metadata, ) end # lock @debug "parsed dictionary batch message: id=$id, data=$values\n" elseif header isa Meta.RecordBatch @debug "parsing record batch message: compression = $(header.compression)" push!( tasks, collect_cols!( rbi, rb_cols_lock, rb_cols, sch, batch, dictencodingslockable, convert, ), ) rbi += 1 else throw(ArgumentError("unsupported arrow message type: $(typeof(header))")) end end end _waitall(tasks) lu = lookup(t) ty = types(t) # 158; some implementations may send 0 record batches # no more multithreading, so no need to take the lock now if length(rb_cols) == 0 && !isnothing(sch) for field in sch.fields T = juliaeltype(field, buildmetadata(field), convert) push!(columns(t), T[]) end end if length(rb_cols) > 0 foreach(x -> push!(columns(t), x), rb_cols[1]) end if length(rb_cols) > 1 foreach(enumerate(rb_cols[2])) do (i, x) columns(t)[i] = ChainedVector([columns(t)[i], x]) end foreach(3:length(rb_cols)) do j foreach(enumerate(rb_cols[j])) do (i, x) append!(columns(t)[i], x) end end end for (nm, col) in zip(names(t), columns(t)) lu[nm] = col push!(ty, eltype(col)) end getfield(t, :metadata)[] = buildmetadata(sch) return t end function collect_cols!( rbi, rb_cols_lock, rb_cols, sch, batch, dictencodingslockable, convert, ) @wkspawn begin cols = collect(VectorIterator(sch, batch, dictencodingslockable, convert)) @lock rb_cols_lock begin if length(rb_cols) < rbi resize!(rb_cols, rbi) end rb_cols[rbi] = cols end end end function getdictionaries!(dictencoded, field) d = field.dictionary if d !== nothing dictencoded[d.id] = field end if field.children !== nothing for child in field.children getdictionaries!(dictencoded, child) end end return end struct Batch msg::Meta.Message bytes::Vector{UInt8} pos::Int id::Int end function Base.iterate(x::BatchIterator, (pos, id)=(x.startpos, 0)) @debug "checking for next arrow message: pos = $pos" if pos + 3 > length(x.bytes) @debug "not enough bytes left for another batch message" return nothing end if readbuffer(x.bytes, pos, UInt32) != CONTINUATION_INDICATOR_BYTES @debug "didn't find continuation byte to keep parsing messages: $(readbuffer(x.bytes, pos, UInt32))" return nothing end pos += 4 if pos + 3 > length(x.bytes) @debug "not enough bytes left to read length of another batch message" return nothing end msglen = readbuffer(x.bytes, pos, Int32) if msglen == 0 @debug "message has 0 length; terminating message parsing" return nothing end pos += 4 if pos + msglen - 1 > length(x.bytes) @debug "not enough bytes left to read Meta.Message" return nothing end msg = FlatBuffers.getrootas(Meta.Message, x.bytes, pos - 1) pos += msglen # pos now points to message body @debug "parsing message: pos = $pos, msglen = $msglen, bodyLength = $(msg.bodyLength)" if pos + msg.bodyLength - 1 > length(x.bytes) @debug "not enough bytes left to read message body" return nothing end return Batch(msg, x.bytes, pos, id), (pos + msg.bodyLength, id + 1) end struct VectorIterator schema::Meta.Schema batch::Batch # batch.msg.header MUST BE RecordBatch dictencodings::Lockable{Dict{Int64,DictEncoding}} convert::Bool end buildmetadata(f::Union{Meta.Field,Meta.Schema}) = buildmetadata(f.custom_metadata) buildmetadata(meta) = toidict(String(kv.key) => String(kv.value) for kv in meta) buildmetadata(::Nothing) = nothing buildmetadata(x::AbstractDict) = x function Base.iterate( x::VectorIterator, (columnidx, nodeidx, bufferidx, varbufferidx)=(Int64(1), Int64(1), Int64(1), Int64(1)), ) columnidx > length(x.schema.fields) && return nothing field = x.schema.fields[columnidx] @debug "building top-level column: field = $(field), columnidx = $columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx, varbufferidx = $varbufferidx" A, nodeidx, bufferidx, varbufferidx = build( field, x.batch, x.batch.msg.header, x.dictencodings, nodeidx, bufferidx, varbufferidx, x.convert, ) @debug "built top-level column: A = $(typeof(A)), columnidx = $columnidx, nodeidx = $nodeidx, bufferidx = $bufferidx, varbufferidx = $varbufferidx" @debug A return A, (columnidx + 1, nodeidx, bufferidx, varbufferidx) end Base.length(x::VectorIterator) = length(x.schema.fields) const ListTypes = Union{Meta.Utf8,Meta.LargeUtf8,Meta.Binary,Meta.LargeBinary,Meta.List,Meta.LargeList} const LargeLists = Union{Meta.LargeUtf8,Meta.LargeBinary,Meta.LargeList,Meta.LargeListView} const ViewTypes = Union{Meta.Utf8View,Meta.BinaryView,Meta.ListView,Meta.LargeListView} function build(field::Meta.Field, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) d = field.dictionary if d !== nothing validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] S = d.indexType === nothing ? Int32 : juliaeltype(field, d.indexType, false) bytes, indices = reinterp(S, batch, buffer, rb.compression) @lock de begin encoding = de[][d.id] A = DictEncoded( bytes, validity, indices, encoding, buildmetadata(field.custom_metadata), ) end nodeidx += 1 bufferidx += 1 else A, nodeidx, bufferidx, varbufferidx = build( field, field.type, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) end return A, nodeidx, bufferidx, varbufferidx end function buildbitmap(batch, rb, nodeidx, bufferidx) buffer = rb.buffers[bufferidx] voff = batch.pos + buffer.offset node = rb.nodes[nodeidx] if rb.compression === nothing return ValidityBitmap(batch.bytes, voff, node.length, node.null_count) else # compressed ptr = pointer(batch.bytes, voff) _, decodedbytes = uncompress(ptr, buffer, rb.compression) return ValidityBitmap(decodedbytes, 1, node.length, node.null_count) end end function uncompress(ptr::Ptr{UInt8}, buffer, compression) buffer.length == 0 && return 0, UInt8[] len = unsafe_load(convert(Ptr{Int64}, ptr)) len == 0 && return 0, UInt8[] ptr += 8 # skip past uncompressed length as Int64 encodedbytes = unsafe_wrap(Array, ptr, buffer.length - 8) if len == -1 # len = -1 means data is not compressed # it's unclear why other language implementations allow this # but we support to be able to read data produced as such return length(encodedbytes), copy(encodedbytes) end decodedbytes = Vector{UInt8}(undef, len) if compression.codec === Meta.CompressionType.LZ4_FRAME comp = lz4_frame_decompressor() Base.@lock comp begin transcode(comp[], encodedbytes, decodedbytes) end elseif compression.codec === Meta.CompressionType.ZSTD comp = zstd_decompressor() Base.@lock comp begin transcode(comp[], encodedbytes, decodedbytes) end else error( "unsupported compression type when reading arrow buffers: $(typeof(compression.codec))", ) end return len, decodedbytes end function reinterp(::Type{T}, batch, buf, compression) where {T} ptr = pointer(batch.bytes, batch.pos + buf.offset) bytes = batch.bytes len = buf.length if compression !== nothing len, bytes = uncompress(ptr, buf, compression) ptr = pointer(bytes) end # it would be technically more correct to check that T.layout->alignment > 8 # but the datatype alignment isn't officially exported, so we're using # primitive types w/ sizeof(T) >= 16 as a proxy for types that need 16-byte alignment if sizeof(T) >= 16 && (UInt(ptr) & 15) != 0 # https://github.com/apache/arrow-julia/issues/345 # https://github.com/JuliaLang/julia/issues/42326 # need to ensure that the data/pointers are aligned to 16 bytes # so we can't use unsafe_wrap here, but do an extra allocation # to avoid the allocation, user needs to ensure input buffer is # 16-byte aligned (somehow, it's not super straightforward how to ensure that) A = Vector{T}(undef, div(len, sizeof(T))) unsafe_copyto!(Ptr{UInt8}(pointer(A)), ptr, len) return bytes, A else return bytes, unsafe_wrap(Array, convert(Ptr{T}, ptr), div(len, sizeof(T))) end end const SubVector{T,P} = SubArray{T,1,P,Tuple{UnitRange{Int64}},true} function build( f::Meta.Field, L::ListTypes, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] ooff = batch.pos + buffer.offset OT = L isa LargeLists ? Int64 : Int32 bytes, offs = reinterp(OT, batch, buffer, rb.compression) offsets = Offsets(bytes, offs) bufferidx += 1 len = rb.nodes[nodeidx].length nodeidx += 1 meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) if L isa Meta.Utf8 || L isa Meta.Utf8View || L isa Meta.LargeUtf8 || L isa Meta.Binary || L isa Meta.BinaryView || L isa Meta.LargeBinary buffer = rb.buffers[bufferidx] bytes, A = reinterp(UInt8, batch, buffer, rb.compression) bufferidx += 1 else bytes = UInt8[] A, nodeidx, bufferidx, varbufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) # juliaeltype returns Vector for List, translate to SubArray S = Base.nonmissingtype(T) if S <: Vector ST = SubVector{eltype(A),typeof(A)} T = S == T ? ST : Union{Missing,ST} end end return List{T,OT,typeof(A)}(bytes, validity, offsets, A, len, meta), nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::ViewTypes, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] _, views = reinterp(ViewElement, batch, buffer, rb.compression) inline = reinterpret(UInt8, views) # reuse the (possibly realigned) memory backing `views` bufferidx += 1 buffers = Vector{UInt8}[] for i = 1:rb.variadicBufferCounts[varbufferidx] buffer = rb.buffers[bufferidx] _, A = reinterp(UInt8, batch, buffer, rb.compression) push!(buffers, A) bufferidx += 1 end varbufferidx += 1 len = rb.nodes[nodeidx].length nodeidx += 1 meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) return View{T}(batch.bytes, validity, views, inline, buffers, len, meta), nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::Union{Meta.FixedSizeBinary,Meta.FixedSizeList}, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 len = rb.nodes[nodeidx].length nodeidx += 1 if L isa Meta.FixedSizeBinary buffer = rb.buffers[bufferidx] bytes, A = reinterp(UInt8, batch, buffer, rb.compression) bufferidx += 1 else bytes = UInt8[] A, nodeidx, bufferidx, varbufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) end meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) return FixedSizeList{T,typeof(A)}(bytes, validity, A, len, meta), nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::Meta.Map, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] ooff = batch.pos + buffer.offset OT = Int32 bytes, offs = reinterp(OT, batch, buffer, rb.compression) offsets = Offsets(bytes, offs) bufferidx += 1 len = rb.nodes[nodeidx].length nodeidx += 1 A, nodeidx, bufferidx, varbufferidx = build(f.children[1], batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) return Map{T,OT,typeof(A)}(validity, offsets, A, len, meta), nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::Meta.Struct, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 len = rb.nodes[nodeidx].length vecs = [] nodeidx += 1 for child in f.children A, nodeidx, bufferidx, varbufferidx = build(child, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) push!(vecs, A) end data = Tuple(vecs) meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) fnames = ntuple(i -> Symbol(f.children[i].name), length(f.children)) return Struct{T,typeof(data),fnames}(validity, data, len, meta), nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::Meta.Union, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" buffer = rb.buffers[bufferidx] bytes, typeIds = reinterp(UInt8, batch, buffer, rb.compression) bufferidx += 1 if L.mode == Meta.UnionMode.Dense buffer = rb.buffers[bufferidx] bytes2, offsets = reinterp(Int32, batch, buffer, rb.compression) bufferidx += 1 end vecs = [] nodeidx += 1 for child in f.children A, nodeidx, bufferidx, varbufferidx = build(child, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert) push!(vecs, A) end data = Tuple(vecs) meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) UT = UnionT(f, convert) if L.mode == Meta.UnionMode.Dense B = DenseUnion{T,UT,typeof(data)}(bytes, bytes2, typeIds, offsets, data, meta) else B = SparseUnion{T,UT,typeof(data)}(bytes, typeIds, data, meta) end return B, nodeidx, bufferidx, varbufferidx end function build( f::Meta.Field, L::Meta.Null, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) return NullVector{maybemissing(T)}(MissingVector(rb.nodes[nodeidx].length), meta), nodeidx + 1, bufferidx, varbufferidx end # primitives function build( f::Meta.Field, ::L, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) where {L} @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] meta = buildmetadata(f.custom_metadata) # get storage type (non-converted) T = juliaeltype(f, nothing, false) @debug "storage type for primitive: T = $T" bytes, A = reinterp(Base.nonmissingtype(T), batch, buffer, rb.compression) len = rb.nodes[nodeidx].length T = juliaeltype(f, meta, convert) @debug "final julia type for primitive: T = $T" return Primitive(T, bytes, validity, A, len, meta), nodeidx + 1, bufferidx + 1, varbufferidx end function build( f::Meta.Field, L::Meta.Bool, batch, rb, de, nodeidx, bufferidx, varbufferidx, convert, ) @debug "building array: L = $L" validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] meta = buildmetadata(f.custom_metadata) # get storage type (non-converted) T = juliaeltype(f, nothing, false) @debug "storage type for primitive: T = $T" buffer = rb.buffers[bufferidx] voff = batch.pos + buffer.offset node = rb.nodes[nodeidx] if rb.compression === nothing decodedbytes = batch.bytes pos = voff # return ValidityBitmap(batch.bytes, voff, node.length, node.null_count) else # compressed ptr = pointer(batch.bytes, voff) _, decodedbytes = uncompress(ptr, buffer, rb.compression) pos = 1 # return ValidityBitmap(decodedbytes, 1, node.length, node.null_count) end len = rb.nodes[nodeidx].length T = juliaeltype(f, meta, convert) return BoolVector{T}(decodedbytes, pos, validity, len, meta), nodeidx + 1, bufferidx + 1, varbufferidx end ================================================ FILE: src/utils.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Determines the total number of bytes needed to store `n` bytes with padding. # Note that the Arrow standard requires buffers to be aligned to 8-byte boundaries. padding(n::Integer, alignment) = ((n + alignment - 1) ÷ alignment) * alignment paddinglength(n::Integer, alignment) = padding(n, alignment) - n function writezeros(io::IO, n::Integer) s = 0 for i ∈ 1:n s += Base.write(io, 0x00) end s end if isdefined(Base, :waitall) const _waitall = waitall else _waitall(tasks) = foreach(wait, tasks) end # efficient writing of arrays writearray(io, col) = writearray(io, maybemissing(eltype(col)), col) function writearray(io::IO, ::Type{T}, col) where {T} if col isa Vector{T} n = Base.write(io, col) elseif isbitstype(T) && ( col isa Vector{Union{T,Missing}} || col isa SentinelVector{T,T,Missing,Vector{T}} ) # need to write the non-selector bytes of isbits Union Arrays n = Base.unsafe_write(io, pointer(col), sizeof(T) * length(col)) elseif col isa ChainedVector n = 0 for A in col.arrays n += writearray(io, T, A) end else n = 0 data = Vector{UInt8}(undef, sizeof(col)) buf = IOBuffer(data; write=true) for x in col n += Base.write(buf, coalesce(x, ArrowTypes.default(T))) end n = Base.write(io, take!(buf)) end return n end getbit(v::UInt8, n::Integer) = (v & (1 << (n - 1))) > 0x00 function setbit(v::UInt8, b::Bool, n::Integer) if b v | 0x02^(n - 1) else v & (0xff ⊻ 0x02^(n - 1)) end end # Determines the number of bytes used by `n` bits, optionally with padding. function bitpackedbytes(n::Integer, alignment) ℓ = cld(n, 8) return ℓ + paddinglength(ℓ, alignment) end # count # of missing elements in an iterable nullcount(col) = count(ismissing, col) # like startswith for strings, but on byte buffers function _startswith(a::AbstractVector{UInt8}, pos::Integer, b::AbstractVector{UInt8}) for i = 1:length(b) @inbounds check = a[pos + i - 1] == b[i] check || return false end return true end # read a single element from a byte vector # copied from read(::IOBuffer, T) in Base function readbuffer(t::AbstractVector{UInt8}, pos::Integer, ::Type{T}) where {T} GC.@preserve t begin ptr::Ptr{T} = pointer(t, pos) x = unsafe_load(ptr) end end # given a number of unique values; what dict encoding _index_ type is most appropriate encodingtype(n) = n < div(typemax(Int8), 2) ? Int8 : n < div(typemax(Int16), 2) ? Int16 : n < div(typemax(Int32), 2) ? Int32 : Int64 maybemissing(::Type{T}) where {T} = T === Missing ? Missing : Base.nonmissingtype(T) withmissing(U::Union, S) = U >: Missing ? Union{Missing,S} : S withmissing(T, S) = T === Missing ? Union{Missing,S} : S function getfooter(filebytes) len = readbuffer(filebytes, length(filebytes) - 9, Int32) FlatBuffers.getrootas(Meta.Footer, filebytes[(end - (9 + len)):(end - 10)], 0) end function getrb(filebytes) f = getfooter(filebytes) rb = f.recordBatches[1] return filebytes[(rb.offset + 1):(rb.offset + 1 + rb.metaDataLength)] # FlatBuffers.getrootas(Meta.Message, filebytes, rb.offset) end function readmessage(filebytes, off=9) @assert readbuffer(filebytes, off, UInt32) === 0xFFFFFFFF len = readbuffer(filebytes, off + 4, Int32) FlatBuffers.getrootas(Meta.Message, filebytes, off + 8) end function tobuffer(data; kwargs...) io = IOBuffer() write(io, data; kwargs...) seekstart(io) return io end toidict(x::Base.ImmutableDict) = x # ref https://github.com/apache/arrow-julia/pull/238#issuecomment-919415809 function toidict(pairs) isempty(pairs) && return Base.ImmutableDict{String,String}() dict = Base.ImmutableDict(first(pairs)) for pair in Iterators.drop(pairs, 1) dict = Base.ImmutableDict(dict, pair) end return dict end ================================================ FILE: src/write.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. const DEFAULT_MAX_DEPTH = 6 """ Arrow.write(io::IO, tbl) Arrow.write(file::String, tbl) tbl |> Arrow.write(io_or_file) Write any [Tables.jl](https://github.com/JuliaData/Tables.jl)-compatible `tbl` out as arrow formatted data. Providing an `io::IO` argument will cause the data to be written to it in the ["streaming" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format), unless `file=true` keyword argument is passed. Providing a `file::String` argument will result in the ["file" format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) being written. Multiple record batches will be written based on the number of `Tables.partitions(tbl)` that are provided; by default, this is just one for a given table, but some table sources support automatic partitioning. Note you can turn multiple table objects into partitions by doing `Tables.partitioner([tbl1, tbl2, ...])`, but note that each table must have the exact same `Tables.Schema`. By default, `Arrow.write` will use multiple threads to write multiple record batches simultaneously (e.g. if julia is started with `julia -t 8` or the `JULIA_NUM_THREADS` environment variable is set). Supported keyword arguments to `Arrow.write` include: * `colmetadata=nothing`: the metadata that should be written as the table's columns' `custom_metadata` fields; must either be `nothing` or an `AbstractDict` of `column_name::Symbol => column_metadata` where `column_metadata` is an iterable of `<:AbstractString` pairs. * `compress`: possible values include `:lz4`, `:zstd`, or your own initialized `LZ4FrameCompressor` or `ZstdCompressor` objects; will cause all buffers in each record batch to use the respective compression encoding * `alignment::Int=8`: specify the number of bytes to align buffers to when written in messages; strongly recommended to only use alignment values of 8 or 64 for modern memory cache line optimization * `dictencode::Bool=false`: whether all columns should use dictionary encoding when being written; to dict encode specific columns, wrap the column/array in `Arrow.DictEncode(col)` * `dictencodenested::Bool=false`: whether nested data type columns should also dict encode nested arrays/buffers; other language implementations [may not support this](https://arrow.apache.org/docs/status.html) * `denseunions::Bool=true`: whether Julia `Vector{<:Union}` arrays should be written using the dense union layout; passing `false` will result in the sparse union layout * `largelists::Bool=false`: causes list column types to be written with Int64 offset arrays; mainly for testing purposes; by default, Int64 offsets will be used only if needed * `maxdepth::Int=$DEFAULT_MAX_DEPTH`: deepest allowed nested serialization level; this is provided by default to prevent accidental infinite recursion with mutually recursive data structures * `metadata=Arrow.getmetadata(tbl)`: the metadata that should be written as the table's schema's `custom_metadata` field; must either be `nothing` or an iterable of `<:AbstractString` pairs. * `ntasks::Int`: number of buffered threaded tasks to allow while writing input partitions out as arrow record batches; default is no limit; for unbuffered writing, pass `ntasks=0` * `file::Bool=false`: if a an `io` argument is being written to, passing `file=true` will cause the arrow file format to be written instead of just IPC streaming """ function write end write(io_or_file; kw...) = x -> write(io_or_file, x; kw...) function write(file_path, tbl; kwargs...) open(Writer, file_path; file=true, kwargs...) do writer write(writer, tbl) end file_path end struct Message msgflatbuf::Any columns::Any bodylen::Any isrecordbatch::Bool blockmsg::Bool headerType::Any end struct Block offset::Int64 metaDataLength::Int32 bodyLength::Int64 end """ Arrow.Writer{T<:IO} An object that can be used to incrementally write Arrow partitions # Examples ```julia julia> writer = open(Arrow.Writer, tempname()) julia> partition1 = (col1 = [1, 2], col2 = ["A", "B"]) (col1 = [1, 2], col2 = ["A", "B"]) julia> Arrow.write(writer, partition1) julia> partition2 = (col1 = [3, 4], col2 = ["C", "D"]) (col1 = [3, 4], col2 = ["C", "D"]) julia> Arrow.write(writer, partition2) julia> close(writer) ``` It's also possible to automatically close the Writer using a do-block: ```julia julia> open(Arrow.Writer, tempname()) do writer partition1 = (col1 = [1, 2], col2 = ["A", "B"]) Arrow.write(writer, partition1) partition2 = (col1 = [3, 4], col2 = ["C", "D"]) Arrow.write(writer, partition2) end ``` """ mutable struct Writer{T<:IO} io::T closeio::Bool compress::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor} writetofile::Bool largelists::Bool denseunions::Bool dictencode::Bool dictencodenested::Bool threaded::Bool alignment::Int32 maxdepth::Int64 meta::Union{Nothing,Base.ImmutableDict{String,String}} colmeta::Union{Nothing,Base.ImmutableDict{Symbol,Base.ImmutableDict{String,String}}} sync::OrderedSynchronizer msgs::Channel{Message} schema::Ref{Tables.Schema} firstcols::Ref{Any} dictencodings::Dict{Int64,Any} blocks::NTuple{2,Vector{Block}} task::Task anyerror::Threads.Atomic{Bool} errorref::Ref{Any} partition_count::Int32 isclosed::Bool end function Base.open( ::Type{Writer}, io::T, compress::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor}, writetofile::Bool, largelists::Bool, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, alignment::Integer, maxdepth::Integer, ntasks::Integer, meta::Union{Nothing,Any}, colmeta::Union{Nothing,Any}, closeio::Bool, ) where {T<:IO} if compress isa Symbol && compress !== :lz4 && compress !== :zstd throw( ArgumentError( "unsupported compress keyword argument value: $compress. Valid values include `:lz4` or `:zstd`", ), ) end sync = OrderedSynchronizer(2) msgs = Channel{Message}(ntasks) schema = Ref{Tables.Schema}() firstcols = Ref{Any}() dictencodings = Dict{Int64,Any}() # Lockable{DictEncoding} blocks = (Block[], Block[]) # start message writing from channel threaded = Threads.nthreads() > 1 task = threaded ? (@wkspawn for msg in msgs Base.write(io, msg, blocks, schema, alignment) end) : (@async for msg in msgs Base.write(io, msg, blocks, schema, alignment) end) anyerror = Threads.Atomic{Bool}(false) errorref = Ref{Any}() meta = _normalizemeta(meta) colmeta = _normalizecolmeta(colmeta) return Writer{T}( io, closeio, compress, writetofile, largelists, denseunions, dictencode, dictencodenested, threaded, alignment, maxdepth, meta, colmeta, sync, msgs, schema, firstcols, dictencodings, blocks, task, anyerror, errorref, 1, false, ) end function Base.open( ::Type{Writer}, io::IO; compress::Union{Nothing,Symbol,LZ4FrameCompressor,ZstdCompressor}=nothing, file::Bool=true, largelists::Bool=false, denseunions::Bool=true, dictencode::Bool=false, dictencodenested::Bool=false, alignment::Integer=8, maxdepth::Integer=DEFAULT_MAX_DEPTH, ntasks::Integer=typemax(Int32), metadata::Union{Nothing,Any}=nothing, colmetadata::Union{Nothing,Any}=nothing, closeio::Bool=false, ) open( Writer, io, compress, file, largelists, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, metadata, colmetadata, closeio, ) end Base.open(::Type{Writer}, file_path; kwargs...) = open(Writer, open(file_path, "w"); kwargs..., closeio=true) function check_errors(writer::Writer) if writer.anyerror[] errorref = writer.errorref[] @error "error writing arrow data on partition = $(errorref[3])" exception = (errorref[1], errorref[2]) error("fatal error writing arrow data") end end function write(writer::Writer, source) @sync for tbl in Tables.partitions(source) check_errors(writer) @debug "processing table partition $(writer.partition_count)" tblcols = Tables.columns(tbl) if !isassigned(writer.firstcols) if writer.writetofile @debug "starting write of arrow formatted file" Base.write(writer.io, FILE_FORMAT_MAGIC_BYTES, b"\0\0") end meta = isnothing(writer.meta) ? getmetadata(source) : writer.meta cols = toarrowtable( tblcols, writer.dictencodings, writer.largelists, writer.compress, writer.denseunions, writer.dictencode, writer.dictencodenested, writer.maxdepth, meta, writer.colmeta, ) writer.schema[] = Tables.schema(cols) writer.firstcols[] = cols put!(writer.msgs, makeschemamsg(writer.schema[], cols)) if !isempty(writer.dictencodings) des = sort!(collect(writer.dictencodings); by=x -> x.first, rev=true) for (id, delock) in des # assign dict encoding ids de = delock.value dictsch = Tables.Schema((:col,), (eltype(de.data),)) dictbatchmsg = makedictionarybatchmsg( dictsch, (col=de.data,), id, false, writer.alignment, ) put!(writer.msgs, dictbatchmsg) end end recbatchmsg = makerecordbatchmsg(writer.schema[], cols, writer.alignment) put!(writer.msgs, recbatchmsg) else # XXX There is a race condition in the processing of dict encodings # so we disable multithreaded writing until that can be addressed. See #582 # if writer.threaded # @wkspawn process_partition( # tblcols, # writer.dictencodings, # writer.largelists, # writer.compress, # writer.denseunions, # writer.dictencode, # writer.dictencodenested, # writer.maxdepth, # writer.sync, # writer.msgs, # writer.alignment, # $(writer.partition_count), # writer.schema, # writer.errorref, # writer.anyerror, # writer.meta, # writer.colmeta, # ) # else @async process_partition( tblcols, writer.dictencodings, writer.largelists, writer.compress, writer.denseunions, writer.dictencode, writer.dictencodenested, writer.maxdepth, writer.sync, writer.msgs, writer.alignment, $(writer.partition_count), writer.schema, writer.errorref, writer.anyerror, writer.meta, writer.colmeta, ) # end end writer.partition_count += 1 end check_errors(writer) return end function Base.close(writer::Writer) writer.isclosed && return # close our message-writing channel, no further put!-ing is allowed close(writer.msgs) # now wait for our message-writing task to finish writing !istaskfailed(writer.task) && wait(writer.task) if (!isassigned(writer.schema) || !isassigned(writer.firstcols)) writer.closeio && close(writer.io) writer.isclosed = true return end # write empty message if !writer.writetofile msg = Message(UInt8[], nothing, 0, true, false, Meta.Schema) Base.write(writer.io, msg, writer.blocks, writer.schema, writer.alignment) writer.closeio && close(writer.io) writer.isclosed = true return end b = FlatBuffers.Builder(1024) schfoot = makeschema(b, writer.schema[], writer.firstcols[]) recordbatches = if !isempty(writer.blocks[1]) N = length(writer.blocks[1]) Meta.footerStartRecordBatchesVector(b, N) for blk in Iterators.reverse(writer.blocks[1]) Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength) end FlatBuffers.endvector!(b, N) else FlatBuffers.UOffsetT(0) end dicts = if !isempty(writer.blocks[2]) N = length(writer.blocks[2]) Meta.footerStartDictionariesVector(b, N) for blk in Iterators.reverse(writer.blocks[2]) Meta.createBlock(b, blk.offset, blk.metaDataLength, blk.bodyLength) end FlatBuffers.endvector!(b, N) else FlatBuffers.UOffsetT(0) end Meta.footerStart(b) Meta.footerAddVersion(b, Meta.MetadataVersion.V5) Meta.footerAddSchema(b, schfoot) Meta.footerAddDictionaries(b, dicts) Meta.footerAddRecordBatches(b, recordbatches) foot = Meta.footerEnd(b) FlatBuffers.finish!(b, foot) footer = FlatBuffers.finishedbytes(b) Base.write(writer.io, footer) Base.write(writer.io, Int32(length(footer))) Base.write(writer.io, "ARROW1") writer.closeio && close(writer.io) writer.isclosed = true nothing end function write(io::IO, tbl; kwargs...) open(Writer, io; file=false, kwargs...) do writer write(writer, tbl) end io end function write( io, source, writetofile, largelists, compress, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, meta, colmeta, ) open( Writer, io, compress, writetofile, largelists, denseunions, dictencode, dictencodenested, alignment, maxdepth, ntasks, meta, colmeta, ) do writer write(writer, source) end io end function process_partition( cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, sync, msgs, alignment, i, sch, errorref, anyerror, meta, colmeta, ) try cols = toarrowtable( cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, meta, colmeta, ) dictmsgs = nothing if !isempty(cols.dictencodingdeltas) dictmsgs = [] for de in cols.dictencodingdeltas dictsch = Tables.Schema((:col,), (eltype(de.data),)) push!( dictmsgs, makedictionarybatchmsg(dictsch, (col=de.data,), de.id, true, alignment), ) end end put!(sync, i) do if !isnothing(dictmsgs) foreach(msg -> put!(msgs, msg), dictmsgs) end put!(msgs, makerecordbatchmsg(sch[], cols, alignment)) end catch e errorref[] = (e, catch_backtrace(), i) anyerror[] = true end return end struct ToArrowTable sch::Tables.Schema cols::Vector{Any} metadata::Union{Nothing,Base.ImmutableDict{String,String}} dictencodingdeltas::Vector{DictEncoding} end function toarrowtable( cols, dictencodings, largelists, compress, denseunions, dictencode, dictencodenested, maxdepth, meta, colmeta, ) @debug "converting input table to arrow formatted columns" sch = Tables.schema(cols) types = collect(sch.types) N = length(types) newcols = Vector{Any}(undef, N) newtypes = Vector{Type}(undef, N) dictencodingdeltas = DictEncoding[] Tables.eachcolumn(sch, cols) do col, i, nm oldcolmeta = getmetadata(col) newcolmeta = isnothing(colmeta) ? oldcolmeta : get(colmeta, nm, oldcolmeta) newcol = toarrowvector( col, i, dictencodings, dictencodingdeltas, newcolmeta; compression=compress, largelists=largelists, denseunions=denseunions, dictencode=dictencode, dictencodenested=dictencodenested, maxdepth=maxdepth, ) newtypes[i] = eltype(newcol) newcols[i] = newcol end minlen, maxlen = isempty(newcols) ? (0, 0) : extrema(length, newcols) minlen == maxlen || throw(ArgumentError("columns with unequal lengths detected: $minlen < $maxlen")) meta = _normalizemeta(meta) return ToArrowTable( Tables.Schema(sch.names, newtypes), newcols, meta, dictencodingdeltas, ) end Tables.columns(x::ToArrowTable) = x Tables.rowcount(x::ToArrowTable) = length(x.cols) == 0 ? 0 : length(x.cols[1]) Tables.schema(x::ToArrowTable) = x.sch Tables.columnnames(x::ToArrowTable) = x.sch.names Tables.getcolumn(x::ToArrowTable, i::Int) = x.cols[i] function Base.write(io::IO, msg::Message, blocks, sch, alignment) metalen = padding(length(msg.msgflatbuf), alignment) @debug "writing message: metalen = $metalen, bodylen = $(msg.bodylen), isrecordbatch = $(msg.isrecordbatch), headerType = $(msg.headerType)" if msg.blockmsg push!( blocks[msg.isrecordbatch ? 1 : 2], Block(position(io), metalen + 8, msg.bodylen), ) end # now write the final message spec out # continuation byte n = Base.write(io, CONTINUATION_INDICATOR_BYTES) # metadata length n += Base.write(io, Int32(metalen)) # message flatbuffer n += Base.write(io, msg.msgflatbuf) n += writezeros(io, paddinglength(length(msg.msgflatbuf), alignment)) # message body if msg.columns !== nothing # write out buffers for col in Tables.Columns(msg.columns) writebuffer(io, col, alignment) end end return n end function makemessage(b, headerType, header, columns=nothing, bodylen=0) # write the message flatbuffer object Meta.messageStart(b) Meta.messageAddVersion(b, Meta.MetadataVersion.V5) Meta.messageAddHeaderType(b, headerType) Meta.messageAddHeader(b, header) Meta.messageAddBodyLength(b, Int64(bodylen)) # Meta.messageAddCustomMetadata(b, meta) # Meta.messageStartCustomMetadataVector(b, num_meta_elems) msg = Meta.messageEnd(b) FlatBuffers.finish!(b, msg) return Message( FlatBuffers.finishedbytes(b), columns, bodylen, headerType == Meta.RecordBatch, headerType == Meta.RecordBatch || headerType == Meta.DictionaryBatch, headerType, ) end function makeschema(b, sch::Tables.Schema, columns) # build Field objects names = sch.names N = length(names) fieldoffsets = [fieldoffset(b, names[i], columns.cols[i]) for i = 1:N] Meta.schemaStartFieldsVector(b, N) for off in Iterators.reverse(fieldoffsets) FlatBuffers.prependoffset!(b, off) end fields = FlatBuffers.endvector!(b, N) if columns.metadata !== nothing kvs = columns.metadata kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs)) for (i, (k, v)) in enumerate(kvs) koff = FlatBuffers.createstring!(b, String(k)) voff = FlatBuffers.createstring!(b, String(v)) Meta.keyValueStart(b) Meta.keyValueAddKey(b, koff) Meta.keyValueAddValue(b, voff) kvoffs[i] = Meta.keyValueEnd(b) end Meta.schemaStartCustomMetadataVector(b, length(kvs)) for off in Iterators.reverse(kvoffs) FlatBuffers.prependoffset!(b, off) end meta = FlatBuffers.endvector!(b, length(kvs)) else meta = FlatBuffers.UOffsetT(0) end # write schema object Meta.schemaStart(b) Meta.schemaAddEndianness(b, Meta.Endianness.Little) Meta.schemaAddFields(b, fields) Meta.schemaAddCustomMetadata(b, meta) return Meta.schemaEnd(b) end function makeschemamsg(sch::Tables.Schema, columns) @debug "building schema message: sch = $sch" b = FlatBuffers.Builder(1024) schema = makeschema(b, sch, columns) return makemessage(b, Meta.Schema, schema) end function fieldoffset(b, name, col) nameoff = FlatBuffers.createstring!(b, string(name)) T = eltype(col) nullable = T >: Missing # check for custom metadata if getmetadata(col) !== nothing kvs = getmetadata(col) kvoffs = Vector{FlatBuffers.UOffsetT}(undef, length(kvs)) for (i, (k, v)) in enumerate(kvs) koff = FlatBuffers.createstring!(b, String(k)) voff = FlatBuffers.createstring!(b, String(v)) Meta.keyValueStart(b) Meta.keyValueAddKey(b, koff) Meta.keyValueAddValue(b, voff) kvoffs[i] = Meta.keyValueEnd(b) end Meta.fieldStartCustomMetadataVector(b, length(kvs)) for off in Iterators.reverse(kvoffs) FlatBuffers.prependoffset!(b, off) end meta = FlatBuffers.endvector!(b, length(kvs)) else meta = FlatBuffers.UOffsetT(0) end # build dictionary if isdictencoded(col) encodingtype = indtype(col) IT, inttype, _ = arrowtype(b, encodingtype) Meta.dictionaryEncodingStart(b) Meta.dictionaryEncodingAddId(b, Int64(getid(col))) Meta.dictionaryEncodingAddIndexType(b, inttype) # TODO: support isOrdered? Meta.dictionaryEncodingAddIsOrdered(b, false) dict = Meta.dictionaryEncodingEnd(b) else dict = FlatBuffers.UOffsetT(0) end type, typeoff, children = arrowtype(b, col) if children !== nothing Meta.fieldStartChildrenVector(b, length(children)) for off in Iterators.reverse(children) FlatBuffers.prependoffset!(b, off) end children = FlatBuffers.endvector!(b, length(children)) else Meta.fieldStartChildrenVector(b, 0) children = FlatBuffers.endvector!(b, 0) end # build field object if isdictencoded(col) @debug "building field: name = $name, nullable = $nullable, T = $T, type = $type, inttype = $IT, dictionary id = $(getid(col))" else @debug "building field: name = $name, nullable = $nullable, T = $T, type = $type" end Meta.fieldStart(b) Meta.fieldAddName(b, nameoff) Meta.fieldAddNullable(b, nullable) Meta.fieldAddTypeType(b, type) Meta.fieldAddType(b, typeoff) Meta.fieldAddDictionary(b, dict) Meta.fieldAddChildren(b, children) Meta.fieldAddCustomMetadata(b, meta) return Meta.fieldEnd(b) end struct FieldNode length::Int64 null_count::Int64 end struct Buffer offset::Int64 length::Int64 end function makerecordbatchmsg( sch::Tables.Schema{names,types}, columns, alignment, ) where {names,types} b = FlatBuffers.Builder(1024) recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment) return makemessage(b, Meta.RecordBatch, recordbatch, columns, bodylen) end function makerecordbatch( b, sch::Tables.Schema{names,types}, columns, alignment, ) where {names,types} nrows = Tables.rowcount(columns) compress = nothing fieldnodes = FieldNode[] fieldbuffers = Buffer[] bufferoffset = 0 for col in Tables.Columns(columns) if col isa Compressed compress = compressiontype(col) end bufferoffset = makenodesbuffers!(col, fieldnodes, fieldbuffers, bufferoffset, alignment) end @debug "building record batch message: nrows = $nrows, sch = $sch, compress = $compress" # write field nodes objects FN = length(fieldnodes) Meta.recordBatchStartNodesVector(b, FN) for fn in Iterators.reverse(fieldnodes) Meta.createFieldNode(b, fn.length, fn.null_count) end nodes = FlatBuffers.endvector!(b, FN) # write buffer objects bodylen = 0 BN = length(fieldbuffers) Meta.recordBatchStartBuffersVector(b, BN) for buf in Iterators.reverse(fieldbuffers) Meta.createBuffer(b, buf.offset, buf.length) bodylen += padding(buf.length, alignment) end buffers = FlatBuffers.endvector!(b, BN) # compression if compress !== nothing Meta.bodyCompressionStart(b) Meta.bodyCompressionAddCodec(b, compress) Meta.bodyCompressionAddMethod(b, Meta.BodyCompressionMethod.BUFFER) compression = Meta.bodyCompressionEnd(b) else compression = FlatBuffers.UOffsetT(0) end # write record batch object @debug "built record batch message: nrows = $nrows, nodes = $fieldnodes, buffers = $fieldbuffers, compress = $compress, bodylen = $bodylen" Meta.recordBatchStart(b) Meta.recordBatchAddLength(b, Int64(nrows)) Meta.recordBatchAddNodes(b, nodes) Meta.recordBatchAddBuffers(b, buffers) Meta.recordBatchAddCompression(b, compression) return Meta.recordBatchEnd(b), bodylen end function makedictionarybatchmsg(sch, columns, id, isdelta, alignment) @debug "building dictionary message: id = $id, sch = $sch, isdelta = $isdelta" b = FlatBuffers.Builder(1024) recordbatch, bodylen = makerecordbatch(b, sch, columns, alignment) Meta.dictionaryBatchStart(b) Meta.dictionaryBatchAddId(b, Int64(id)) Meta.dictionaryBatchAddData(b, recordbatch) Meta.dictionaryBatchAddIsDelta(b, isdelta) dictionarybatch = Meta.dictionaryBatchEnd(b) return makemessage(b, Meta.DictionaryBatch, dictionarybatch, columns, bodylen) end ================================================ FILE: test/Project.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. [deps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestSetExtensions = "98d24dd4-01ad-11ea-1b02-c9a08f80db04" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] ArrowTypes = "2.3" CategoricalArrays = "1" DataAPI = "1" DataFrames = "1" FilePathsBase = "0.9" JSON3 = "1" OffsetArrays = "1" PooledArrays = "1" StructTypes = "1" SentinelArrays = "1" Tables = "1" TestSetExtensions = "3" TimeZones = "1" ================================================ FILE: test/arrowjson/datetime.json ================================================ { "schema": { "fields": [ { "name": "f0", "type": { "name": "date", "unit": "DAY" }, "nullable": true, "children": [] }, { "name": "f1", "type": { "name": "date", "unit": "MILLISECOND" }, "nullable": true, "children": [] }, { "name": "f2", "type": { "name": "time", "unit": "SECOND", "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "f3", "type": { "name": "time", "unit": "MILLISECOND", "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "f4", "type": { "name": "time", "unit": "MICROSECOND", "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "f5", "type": { "name": "time", "unit": "NANOSECOND", "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "f6", "type": { "name": "timestamp", "unit": "SECOND" }, "nullable": true, "children": [] }, { "name": "f7", "type": { "name": "timestamp", "unit": "MILLISECOND" }, "nullable": true, "children": [] }, { "name": "f8", "type": { "name": "timestamp", "unit": "MICROSECOND" }, "nullable": true, "children": [] }, { "name": "f9", "type": { "name": "timestamp", "unit": "NANOSECOND" }, "nullable": true, "children": [] }, { "name": "f10", "type": { "name": "timestamp", "unit": "MILLISECOND" }, "nullable": true, "children": [] }, { "name": "f11", "type": { "name": "timestamp", "unit": "SECOND", "timezone": "UTC" }, "nullable": true, "children": [] }, { "name": "f12", "type": { "name": "timestamp", "unit": "MILLISECOND", "timezone": "US/Eastern" }, "nullable": true, "children": [] }, { "name": "f13", "type": { "name": "timestamp", "unit": "MICROSECOND", "timezone": "Europe/Paris" }, "nullable": true, "children": [] }, { "name": "f14", "type": { "name": "timestamp", "unit": "NANOSECOND", "timezone": "US/Pacific" }, "nullable": true, "children": [] } ] }, "batches": [ { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ -719162, 2932896, 1251583, -354578, 1947695, -669151, 26653 ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-62135596800000", "253402214400000", "-2820221740189", "71549882314362", "125067746235030", "-27038791348254", "42137473450326" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ 0, 86400, 76127, 6700, 27103, 56151, 56654 ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ 0, 86400000, 17984286, 76370591, 60937117, 2240575, 8788989 ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "0", "86400000000", "74085029005", "23058796418", "5827157574", "30753386088", "41165364667" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "0", "86400000000000", "23907445718784", "74345421086418", "75233481254444", "82172159793710", "58497242525071" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-62135596800", "253402214400", "-61131551586", "109841560561", "-3916465142", "146694684650", "138850275868" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-62135596800000", "253402214400000", "73881152631437", "69908713976427", "252339755754438", "-24746530024729", "169302540975380" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-62135596800000000", "253402214400000000", "201464679490582249", "168126161155504013", "-23403778250906066", "205706554937392102", "45776665091115087" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-9223372036854775808", "9223372036854775807", "-6357255048670867880", "-8661447973993819541", "-8212788386909103318", "-8530954041419345600", "-4218486829304453721" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-62135596800000", "253402214400000", "206983911256609", "94973546379549", "-18473207641060", "36529119814530", "143273969098011" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-62135596800", "253402214400", "225567855249", "18090198256", "-18645631593", "1683299996", "240974238031" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-62135596800000", "253402214400000", "-41888047432132", "142167692985706", "96450253340232", "-28800292871111", "31551906541089" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-62135596800000000", "253402214400000000", "-55035741541368439", "110555619232926841", "13584197914180451", "-40099398122556776", "154575532939365500" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-9223372036854775808", "9223372036854775807", "5175948389020765869", "5557679156666679724", "4250919303876106324", "9160676477011889469", "8585006913301874724" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ -719162, 2932896, 1879965, -566367, 37728, 1761040, 279144, 1056794, 756303, 525725 ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-62135596800000", "253402214400000", "-38092110755085", "-28445105640862", "82150583163219", "54419670636015", "157522888788052", "-41135178703404", "-55692081078291", "23161948344048" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0, 1, 0, 0 ], "DATA": [ 0, 86400, 68158, 43645, 82390, 63272, 8687, 73663, 41080, 16606 ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ 0, 86400000, 4300160, 71414742, 77204018, 20632726, 31365614, 66601445, 59573489, 62138475 ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "0", "86400000000", "28651881349", "76962235570", "72557141940", "81551970477", "20740172257", "69927019990", "76743031592", "82821335874" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 0, 0 ], "DATA": [ "0", "86400000000000", "69757112637445", "57552711513942", "29426946816946", "34878855922929", "33344575898987", "80887006335433", "34037765279999", "51577535310194" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-62135596800", "253402214400", "17362133914", "-45891238920", "184875963653", "189805054827", "-58368591641", "11979945774", "-42159999942", "-40114167869" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-62135596800000", "253402214400000", "228991365143433", "169961535994833", "200469360821110", "69234108321383", "198319780924526", "191497586634193", "175427870270356", "57342673854963" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-62135596800000000", "253402214400000000", "56980200145982394", "181929648203745781", "139102923751831867", "208997257530770666", "16136961511557279", "41555612864958844", "13419848118557598", "87383692083185618" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-9223372036854775808", "9223372036854775807", "-7031715621840828119", "6059066797068981521", "-1646261793033501417", "-4220806875454311426", "-5197114782094970725", "-8786449967831538943", "2974021310284646715", "-8065049992539820014" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-62135596800000", "253402214400000", "178310981489839", "-57632494932898", "-49036210908585", "52895117552494", "171282515196488", "132014017559614", "37589110284897", "-58565057255450" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-62135596800", "253402214400", "-25861647763", "225566577159", "239974631847", "9594019763", "195861877033", "-19961060193", "75621579368", "-37516489502" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-62135596800000", "253402214400000", "197770622812426", "195993483135189", "187243903796151", "193213107363200", "83570298570259", "252571502045214", "129428288356579", "-6553516468568" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-62135596800000000", "253402214400000000", "227405247332731417", "-23876629555725661", "172967389442803645", "214366955371313241", "86933375268516953", "162567381239071692", "40270626452354761", "-9766478375147980" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-9223372036854775808", "9223372036854775807", "-3367778002617009344", "-921482854487607004", "1280046021732622411", "-2084512789553264359", "6774927372092824293", "2301253808511314195", "-447823511949250637", "-9137589927882857097" ] } ] } ] } ================================================ FILE: test/arrowjson/decimal.json ================================================ { "schema": { "fields": [ { "name": "f0", "type": { "name": "decimal", "precision": 3, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f1", "type": { "name": "decimal", "precision": 4, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f2", "type": { "name": "decimal", "precision": 5, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f3", "type": { "name": "decimal", "precision": 6, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f4", "type": { "name": "decimal", "precision": 7, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f5", "type": { "name": "decimal", "precision": 8, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f6", "type": { "name": "decimal", "precision": 9, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f7", "type": { "name": "decimal", "precision": 10, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f8", "type": { "name": "decimal", "precision": 11, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f9", "type": { "name": "decimal", "precision": 12, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f10", "type": { "name": "decimal", "precision": 13, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f11", "type": { "name": "decimal", "precision": 14, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f12", "type": { "name": "decimal", "precision": 15, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f13", "type": { "name": "decimal", "precision": 16, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f14", "type": { "name": "decimal", "precision": 17, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f15", "type": { "name": "decimal", "precision": 18, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f16", "type": { "name": "decimal", "precision": 19, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f17", "type": { "name": "decimal", "precision": 20, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f18", "type": { "name": "decimal", "precision": 21, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f19", "type": { "name": "decimal", "precision": 22, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f20", "type": { "name": "decimal", "precision": 23, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f21", "type": { "name": "decimal", "precision": 24, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f22", "type": { "name": "decimal", "precision": 25, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f23", "type": { "name": "decimal", "precision": 26, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f24", "type": { "name": "decimal", "precision": 27, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f25", "type": { "name": "decimal", "precision": 28, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f26", "type": { "name": "decimal", "precision": 29, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f27", "type": { "name": "decimal", "precision": 30, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f28", "type": { "name": "decimal", "precision": 31, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f29", "type": { "name": "decimal", "precision": 32, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f30", "type": { "name": "decimal", "precision": 33, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f31", "type": { "name": "decimal", "precision": 34, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f32", "type": { "name": "decimal", "precision": 35, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f33", "type": { "name": "decimal", "precision": 36, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f34", "type": { "name": "decimal", "precision": 37, "scale": 2 }, "nullable": true, "children": [] }, { "name": "f35", "type": { "name": "decimal", "precision": 38, "scale": 2 }, "nullable": true, "children": [] } ] }, "batches": [ { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-14265", "-3301", "-20359", "5259", "-10833", "-19794", "22904" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-7020", "7396", "9702", "-29862", "25915", "17934", "23441" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-6799031", "-6350361", "-7919057", "-2688856", "-2460218", "-5498780", "5580928" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-3533649", "643928", "2739361", "-2006582", "7501109", "-2029241", "-6554749" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-1383097445", "-317364446", "39137303", "-203304373", "284864794", "-1902924832", "694509255" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "1445584597", "-1479256303", "1986678586", "653640925", "-1377647126", "163472005", "-985666433" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-1032313347", "1968833128", "2108909581", "1727353475", "912414766", "-635823470", "1712241290" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "154945363887", "416184610386", "339640718067", "517615365091", "-225329280656", "170183803130", "-151819857736" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "415862710871", "-497367616337", "-513589024815", "365532163027", "162771025458", "-217545761908", "-177764161272" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "139749710369981", "-83794245890178", "-91856283653391", "136480936919831", "139751556918685", "-128711073562986", "57528145123438" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "52917228652967", "-94943960868145", "-95787469176401", "138513251818631", "72391840904205", "15456369093039", "43627762817987" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-55939733680307", "109774177882810", "-18751094013174", "66217182114674", "54552088012955", "10184533351464", "-42623419179005" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "5323426016158598", "-14889118959500841", "17725275930019497", "8976278451851381", "-13047873474817232", "-35698038632707430", "35553131382011652" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 1, 1 ], "DATA": [ "22248347423524568", "21382628128477388", "10729166497278728", "33552782977485637", "-27049764851522362", "-22662588671534830", "34450594992450092" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "1793092340943680288", "6687492261570159758", "-8215335212004105263", "4478178378408252470", "163214921176454093", "8663994206540321487", "-7336047061160684053" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-8553613303342920727", "2303004565228604365", "4387581718276767427", "-9190240538897215152", "-2359611618219705826", "-4394705260138635628", "-5098494694180487526" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-2066857943452927830158", "903014216615599800157", "-550458216531247342288", "2017634601926315282348", "848752616371714143556", "1142061465959813542452", "1385502695475659971908" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1 ], "DATA": [ "-2114118262682166206082", "1902359162690886678168", "416686278682682920537", "1670406372017974831657", "1672178198289799650411", "1290828393749566925129", "477509832520793484657" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-761286776745227142069", "-519173571795485925518", "-1152371818412313184820", "-1693854308571834153795", "-2338147549154180008842", "-722980793690531284755", "-308453791763555488903" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-363139155978036972417607", "-67592154020748584606597", "229641197562527780640996", "-346708211516644122649482", "-1250859224618770910347", "-324094365896255288686912", "587754147918610672847722" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-260044582391618089260319", "-571174996847059618302630", "-109712997473546598340520", "-504887878361994310822232", "-79933142040493800702044", "391593215117382882517477", "253411954510878438009626" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 0 ], "DATA": [ "78355683904145007775288840", "-119384708322041630321535990", "-100920737167981116982294279", "83826802691186830123887155", "-142861875795075155563507787", "-145798388286082279657253594", "-25329913740178438718347749" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "2728894406873273319876599", "-26824077854062930008319059", "59282342970725803527905051", "-102858143095488614827267122", "-86746033992194633744162126", "151758737255322612002715481", "-7882766239372883938103148" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-79786812518583385435068134", "-60631765880118763666388680", "-6864681441231111204611249", "13050926774682302829016376", "-70829524080848069150697712", "-82269446396753351313982437", "-64035656687419899366647308" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "3955725931924121460907631531", "-21274213085809772733586643935", "-12093030231048856037056919233", "1591415318816864321772119800", "15686477033795236409156083537", "35545421326820517900037881163", "22425766368694890650567525141" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "29018707613584242707294878326", "17791141760763489912825555133", "16531138885547875931113958621", "-30513053118126324165999828125", "15960846333545963474586001659", "36917332355702074649716398096", "-23564003365919469704120973292" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "9213099815853647106837374141135", "-2927321977169069503340222047887", "6334956127042213437236348937051", "421008034487997672364705982278", "-8216535847017377542720322569325", "9010595708854142678424220967811", "4188490944857456828411574213081" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-8400462145843923761912990483225", "-9532520685799568447195818877630", "6941627626857671447061360399506", "-1279966284289260405732779325351", "1498483597016609631513149683949", "-1529118713555007787773854671666", "8107117611898163619507416633696" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-2915756808824546582752189119657", "689849096940399949037495607644", "6949792968358403147068782203692", "-9209851065708847333103701669232", "-9846735769797668003358129310662", "-828325389508874726576445634733", "-548287410953591263203641276502" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "2329344668638312257280241058269271", "-752344765844397811032529707212356", "147451832276294483763966453842451", "2043297198679260475716881353145488", "-1700852196536476328772217191241342", "395231723125832272118398102832808", "1021064751191444192162606435133102" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "1059349538643979619795009909676873", "1077517333972932491341963859241254", "-1269329697335528941024149247301687", "1492334714292199711015081732157237", "1164940978846205931486582828955479", "1226701026828005364221866490976254", "-2246642220670050837117291416085961" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-453427466718461089284644212936857043", "401968647744354893846373970134628607", "296564973980089833086319484394180932", "300255396607176659100995955356045655", "-128210286701929565620215086982795175", "-76575845460847329054975157675278272", "150610860039362064995272346289079343" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "421046835537097379208584523685561537", "-12336249922904854000276177315936166", "-643245709818789885569562430725515554", "28125849177083902136768190312091645", "-144070901322770869842559076535428168", "404632889001498427711768281303970316", "484413224979251839912542328701460679" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-74076465584946023556734876157740408561", "48775465105440153520372738047296715437", "-27256511570535798216063178902711504522", "17556306662405983159878112274642973563", "-100278707484244689483962841662232621044", "61342616787415403733699116416597761322", "65469616326882669620827062523697818819" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "160988666415816646705442929165000844171", "-150079614246235114587531344502672720143", "130458754430497218349443163826152965786", "162593370774960128510764087469483971628", "100335479568560073327353331226427021396", "154469868317889131666703500879972187378", "109851602321776291877341624608658829262" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "90507703396950205824782315866890653146", "33101701517846982239639071710846694736", "-162364169137146690363873495754205930932", "6552361136224904568043204720462599330", "55934073603932057089039430724186991179", "-40518421289171987097923903446970623529", "21964873705187790865866750301783750442" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-4326", "-4742", "24127", "-28455", "-25093", "18893", "6010", "18847", "-29502", "32119" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-4350", "6406", "-22249", "-3026", "-27892", "12582", "-26038", "-26810", "-10412", "12059" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-5437050", "-2975935", "-2708551", "5905122", "7598831", "1837938", "3161296", "4144482", "7971237", "3812932" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "8307058", "5437059", "7848415", "210888", "-7438002", "1251661", "1558298", "-8295786", "-1337540", "4329127" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "2143281149", "1036205716", "-483960365", "464783350", "44702295", "1500496151", "2007499481", "-1980167635", "-491561248", "137927958" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "385115557", "-468314229", "-1806182189", "826968059", "1973820915", "-1678321968", "282860534", "-429116070", "-485370474", "-1175964348" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-1660401943", "858637970", "1954585251", "896247082", "-1324508546", "1856733620", "-508486488", "-2144890404", "-595464217", "-541883970" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-475292084638", "-151527930749", "-136244496864", "475724641544", "490808185901", "-380524076056", "347742169602", "-208359763265", "500407573029", "-390649746248" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-463878565883", "-440350084389", "-489309228695", "403236358497", "22420180364", "461550541832", "166802945594", "-75395045157", "349097107753", "-31923850582" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "13351341741346", "132672835377977", "-104121852651737", "-116578873884859", "-93368786381550", "-130866514927019", "-130245922532357", "119422720388976", "39514069962031", "129449354245745" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "117562734639864", "-69335041870511", "88065868490266", "-88825080775852", "-33000609471354", "45253821354104", "-19572766862990", "-20998291433421", "-138710717300131", "105346771642837" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "23613726765605", "-122540956874454", "43298244246934", "38914627030131", "-96822287655033", "-117561922155895", "-116901550232631", "-33238646153535", "87255230245952", "-103811103056354" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "14326292611678389", "11679216137082255", "30602968082225040", "-35083463698552038", "13793460528237121", "29475754221579432", "11956818381525770", "25264515687485849", "-9442305734363215", "32709604673194504" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "10752127983223695", "-17671172332493417", "23204581064269137", "25912317958066911", "-6684281122027701", "-5393708227406178", "-705876736520327", "-8271615012961841", "-2359308340581953", "-10178212042804721" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-785858494820999025", "-1596606269767712061", "-1418974761760001783", "5406463592868106826", "8383949914321587435", "-8953833508549969767", "7221522693757617783", "7517732845880583602", "-919064621805709634", "-5495596733594283265" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-8483275882311463690", "8973190785377545539", "5542339023127975216", "-1516710247958149161", "-6505879801430717005", "2140391223240971073", "-582269641777058598", "-5184242226957355683", "5206992463328246120", "-6079741749013867959" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "1593226799266187966947", "-1812161934828717889262", "-2122590131716118763227", "-1027437168205961683538", "1846540008444089681004", "-1156292451907349126710", "-1957435108076953499486", "-950458759560036353191", "-1125440584483224915212", "2247219347276423832973" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-635230024201890240957", "180963226287776474583", "-1878152791054730240606", "-1904422293952570992101", "1052835439226626094544", "-1858821189756108906198", "1853668161015001150059", "511899624049166705494", "610084498432376101375", "-1029828641365425116105" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ "1885658882244978382784", "565125911580367694969", "-1652560178353597573029", "2133173545067165238416", "-486940365592293262895", "-2060709217874894710408", "1586493850244052619259", "2178470762303491581080", "-1293832362817784170345", "-1858110114261315174517" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-369225411953896208185462", "-432125813663454581808312", "-144256737060926287909147", "437531850902150565537267", "-135152047695882388519227", "200989985671128391254839", "412064182108667107703143", "155783836458581176854752", "-227172778832455276260133", "212767460062823457968604" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "403503153546865033134324", "286440584824595490396339", "191634056603308191156299", "530416421236246257260161", "-317587706869080743041040", "-171530648886789759204250", "222798032246148363279215", "72219173848107712300756", "-281960273386152702267354", "-495494776734887426338667" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "122137352759155073399221518", "-78109201384474477075454853", "-119068241308521301444272353", "-108756965225823937786847241", "-15428358494009873779245484", "19614899796809003767318009", "-115295286224016867112586811", "-8633742807795124483538483", "-150410722825301769338372723", "124250719445278286695627796" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-99183180993590467578076798", "-36808132925962854413539892", "-125679607816807665333773367", "-136622111609623584591267546", "-17266188400280841341905013", "-71769330665427864647331885", "-145814031084605056753603019", "-125056942233093610827766296", "9249166279701860198655516", "126878824313892664602568663" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "69744825329510933906746340", "-30122586028854363823971329", "22212149164944035060004867", "-25973928380545009708020589", "50567555142199657207098957", "-40021098669624532176421808", "-33330934345289112585983041", "-124012214741805393399462125", "-138226312121129798637686830", "2273005251425309465489619" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "26842282533862498382939405997", "17787994211657168910392131484", "7523952681100831594207033636", "18257283108725825658512455316", "-15469531382712880431453128966", "-5009369258227088832031787745", "-35598690213497115663830724084", "22246953732785347649151349772", "9330915514049730010034508310", "-26800441656550330938009306707" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-36331212919615640889272411104", "36192450254329815033783240839", "37749265523377026412760210674", "1776086174556031952147671519", "15362218353534633991283119679", "34413146026389235392142970662", "14292777901465553588604435837", "-17090082469158479303098433453", "18688801441734020761551793736", "-23859805527670127110643731069" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "2220103303731789573334199701663", "6237111177014715580774407162796", "-6338471790385983381447055194941", "2250388387229112362557012222382", "-3919498269201660475165992143963", "-1799934177913445118487985955729", "-2323304519983949754039480799426", "-1301131662575316944670816531905", "-4265160933868529886993472786495", "-4292953046034527295812111533975" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-3484925219213002239805523064883", "6528552859477682991951516050785", "-765962107198382441283095455373", "-1397137479831402026027607048248", "3961682476407229854971103624981", "5975195883181022585096451556775", "-9676800477604513004081790052790", "-5703219619447316623534739055181", "6129137758274201914352345247446", "771936840958552475578155475366" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "7726836802487400261012835642739", "-2899737712900797452349686903367", "-4699841764135292946953397202917", "7418179251651709234455608105088", "8120397131521471010221697932900", "6526213244777384369820937871939", "-3774567743925527043971960901589", "-8873736034246294395374537274419", "488289844645088756680442972516", "-2388144175550709792515001316277" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-47477340538926222211104914237698", "320639937921939986635581989855928", "861736289842228881011999521004060", "-1205256013177842982651274099236361", "-39643995957385740378372042150003", "-1210790336528949301250817830946444", "1182423445445031739603473987778350", "-640337212803879852231150942968813", "-567587726713873060654848198630762", "280641360776654488558911759135115" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-866285575875610221959113146637304", "2344770413948196340883160931087052", "1889857580890564350097047877514790", "-2568726710340227178480991779371723", "-2485213596075291138559974328614959", "-62378076407090361966505761644273", "-2113433394374145039336066697772466", "-2389429054181728925012886258903305", "-2525262726376576108925615177598533", "-604526323558062739370715910785200" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-411175611485478784812936027850047826", "457804241550769132910430204506150717", "1549607974636127184352732575182742", "390025168459008888512891394303962448", "-490664276836807094735857019444661312", "-450640457243898505844766529165221166", "-553339316764622659577593048830066531", "19986969431028149966548563329309114", "512190967497020823088167980283378342", "-587244163212803971173675798133411979" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "246512063483920590600659938977719645", "483324212708049237205831199756205159", "195186088040813738593870463346937474", "-331572691964567402179138728488891845", "466186594735574058231047101478621597", "80512186957271594252263140708656889", "-621018503026533776126752549530840325", "-484705410290628923401085082564299400", "-322852670767952007714303037031011136", "306969564371703225579907932616560177" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "52513592802194657300546985265901321701", "32396354459564619210286503831104805894", "-152800597997887046528471654246818061800", "16886738551251131147537573820997523099", "-108529890244550965992486427743581407764", "-107805100101953838884357149548341742683", "-152428185005223769908895650096091179800", "38697638049920608411728929261890502602", "-53183835204559125450983905289890638199", "-25198915522830164616311861007733226015" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-29473659368749332611063061562898178665", "-64181392425515474831866849063339846755", "90557155313004461723928910484647118568", "69982410088115917703700629187142714952", "137281427883875761429267863353473899975", "66221749426872522877433672036305282198", "139898216716793863940328165403822713329", "-17999019985160863257629297875199464798", "74582888771441381839018581558057879959", "153431389525940565745457493557728519648" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "132130437221517226508875210221998086214", "-53955540660976668373224815268780394481", "-94671974318719360571824313264981742616", "75940549933612625873217731049581943788", "-58970299111050400318748873232294460623", "106861808580585226388022589188291458328", "151810935532684619700572858784852122078", "147244409300271119764509412340177481716", "139138676242120433411406371456479693081", "58927563101546159870009467050083778558" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "25114", "-12925", "-1074", "28646", "-16569", "-16770", "-27151" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "26533", "12085", "25312", "23445", "13356", "-2369", "-29785" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-4380859", "-157657", "-4200925", "6061632", "1972423", "-3016747", "2864606" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-7045373", "-7371800", "1427921", "-7601068", "2525157", "-6433403", "-2477014" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-141345816", "1543059503", "1211501603", "369888761", "894246844", "736166799", "409408642" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-1553193945", "1491657644", "376067040", "-1100453955", "1763626627", "1237652218", "-1680624462" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "102290167", "1044278987", "501196467", "32275276", "-1384353672", "594927697", "-1482240101" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "431716525885", "-396649752792", "-252538061544", "148825582538", "536082535524", "-135755487371", "425356768774" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "137916147840", "21393720222", "-325824439981", "278743651518", "-49501503483", "-419736800011", "503854890502" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-137642247974694", "-126053382015273", "32896127803947", "17450989064436", "-120690899531858", "109217312768874", "-79259001409277" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "114207744387703", "-79306413416767", "139536513022560", "-98423452444221", "78791182271051", "-35285009371089", "41940974948329" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-27264096490547", "136834819442083", "-27088394932437", "-98987047379901", "125550952272750", "-104804800493942", "8768888141448" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "23677368293754508", "-30279797614274598", "-30317510840672041", "-3538291640098903", "-2087381848617356", "-23713764981307104", "-11713886893415435" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-26406417185878787", "-14282719185754107", "-34042541212866444", "24219940349910515", "-23950715983191837", "9125287296093962", "1340350995572397" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "1175386256159753067", "9132036865814002849", "2073302245565101007", "5198424551239400626", "1162180176286851142", "-5469754044264044482", "6861255327733562384" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "6025723723135521005", "-7217621567135194958", "-875939541579707257", "8236552400788004405", "-4353971909177851416", "4189966288831554693", "-2170856197554954993" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "1984509802579054321521", "-1041998696764751045250", "501861238173417097382", "2266100162043156929049", "1092677900985431385639", "2290545820208575246231", "-1929677784513148966088" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "-2343674427840265383765", "2230129800929309514822", "1298773822649292467354", "-1658379353955811152293", "460872411226138227967", "-366025556585308681996", "2303266251171038691928" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 0 ], "DATA": [ "-1553211803508188723315", "-307288386314104261659", "-147859719179575239406", "316737260848156543682", "1901202079912459192862", "-2066795118493461325326", "1097271871910371248765" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "60159431599803576006229", "-349523214568312595512352", "-479410413534686721947634", "-438205000235664595511857", "71434087716492159731908", "313701681861148862347175", "592529110781630294785385" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-41073349320046663529611", "-460196612402794801624059", "-85219737286050217930644", "-430653003445193867062723", "212772177419770007462828", "533773505614581994442892", "588510537528110040152211" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "31643252440554433191940460", "-93699577774061168478202291", "24154537866604719834628304", "30696729192090200230120983", "-99615616997150346574018879", "-570384960873190266772386", "60124053065001805618112484" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "43634928021314649415598611", "-71376931904073326185512276", "-24045621000779198743707062", "6686912739698933453916262", "-154458317914070878667147081", "-48373280695426803475799028", "76169691954204618596160986" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-114582957647416816237217281", "-12748630690377976467956388", "111731504296816655416680693", "4019447671331190931268009", "82801431291231910751983771", "108497088164927504518108753", "19137086084026561540969316" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "11411634354069865542863757714", "4367992064855387626281818334", "21423224512954024770097152486", "-32916059460547773682793752250", "-37598962143905700881222660514", "-26791606874412750044572561764", "-1640980913390047815572783266" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 0 ], "DATA": [ "34745452020371732029340755584", "38633937182521363276851366299", "13907386367292796497248233421", "3516712513415277325145153684", "3056849635253718194064084992", "4551068805171087267034288791", "-17898292752703455443559461457" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-782556900320991870801777710984", "1691629456944775593578700272292", "-3817502570462777233621604475255", "-4582592814198566350977961347647", "-8757778015646360699558792162096", "-3372484932226680791305443517921", "-6773601661940427910934854617600" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-3034111110316845464241842333639", "-1781275911958673372639492265462", "-1931637931243318072822423804991", "-2889899890039585265079105752268", "8441716411760182969194644963692", "7571628885131553956380784504990", "6417570389157484165557130988430" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-8637275815739683704408877390855", "-5205784754631510403458931812287", "5903473813623392609711242445408", "2196392642166569937732303950298", "4322897470867933044080337659838", "-295215953845026604294742492576", "-9707682435519997757567927666374" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1371746941161753233161616198879974", "1982136493212029919715565342281105", "-1917002754487970574686778478783266", "-57865905347137694987990288364929", "2249945384795180946501174810954490", "-428951216116914987102881673111066", "-2274002923925825152016895839891593" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "1092814442698298966273488282610980", "1144413745884554568972739530468097", "-75300812750583864582975138768425", "789050129071516591209015420836704", "-1164029127634305816704854164256453", "2432154715761013584642826292686689", "-1892357160522293273795945838702878" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-122183630549108645927458587184832899", "-243792853164469971347092559188097429", "586041069230257431808895634457019729", "643508290616316791666365172242311322", "140267869100082091472571196986321703", "367901398863058047260102259430609991", "663427402521611050810057766609452801" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "442484503124860279043087511624488704", "-281538291396564587147893059598980836", "-346402069367877555608660697944397703", "121136520298204023131218573046733183", "-297635365268234967618402017598530084", "-374122180189017501660740031013019545", "-581737813428791636660311570131585053" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "133154724605091554639809801656916837785", "75830729335101076443136841481038070489", "120313524491479335277638759335459890905", "61287945625316584877200472317882946434", "101401636551502086495943764753580195892", "-67491973819108763200100227319414962675", "1469727002050376143004735317308620121" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "163689960486968830624238755496353069266", "120222413117901199440645458344006405188", "48580226284038590005071569547583521787", "8088270627535987928187148843962238997", "81865070062931483309512859489282733063", "-10237210340155912860809619233603401462", "83075135112485276897941678118419444602" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-165871274022360766986351083915813500328", "-8686579665026772053190367047771284024", "-159204341085506490972656979604179139282", "109873262356509061148465005250720029433", "-106075570921623241101438726440228153847", "94260033293871207317716609052204748035", "-20194157317096074536517680199183372407" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-30954", "20758", "27427", "20450", "3843", "-28014", "-9284", "-6624", "17505", "-15123" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "685", "-5310", "-781", "-19286", "-2510", "-7200", "3827", "-1198", "-27211", "20037" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ "805128", "-8334205", "-8322011", "7348287", "-2733195", "426429", "3846540", "-6903563", "6889801", "-7512693" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "2488446", "3102298", "-476499", "-2639540", "3282590", "6092430", "-1675823", "5601386", "1196495", "5499393" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-30691203", "661960132", "-1935743351", "1575466801", "256013883", "1160657488", "-777972530", "-1493606470", "486831696", "-523869499" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "1802142466", "1751466925", "1524761681", "1127842279", "1711088497", "993428343", "1169779239", "1513015569", "-1133724977", "2014476536" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "1960388178", "1348475329", "862499131", "1640384591", "-1548686121", "-109978873", "574685145", "-1936942220", "-828064162", "-2111967247" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-10546471442", "410574147711", "-159310325390", "-342068240706", "268451689904", "-188950021888", "174863595081", "75886283067", "204533910683", "-236375793932" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "395399290333", "398808017348", "470458425747", "544514539160", "106030474107", "-220167092355", "403504352884", "-215936236845", "157621638402", "-84528308243" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-11872483439739", "27198663837500", "24092599125599", "94647109181593", "-25234549831778", "18418685805939", "-48015201340553", "94192389246447", "114313155048141", "-8129249986173" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "77060681503342", "-17103335031331", "62085210064375", "-4113774722062", "-39621802698178", "82862411524800", "-80722206760975", "-71853038428164", "74451847897808", "138561565698569" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "11425892925558", "-1067102949941", "-67195834692311", "132254465032616", "81094671882564", "-16512810337582", "31369523023438", "-25612899826767", "-92714617670958", "11132766799365" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "31425051479846585", "23658872883751119", "-22841020643211561", "-5901149141492493", "-9081601009126169", "2009456704881039", "-5853653315964572", "-17548466661612427", "31243548103064502", "-2990459100082881" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-12114077651218054", "34893217784362181", "20057977432890552", "33560592225527770", "-6087365862911550", "35528436105980777", "-18908989690611601", "-23450050973056772", "-24497468547358805", "35295281517022055" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "7512947852738829754", "1506648420706893967", "5351706185125823777", "-8979604389662071296", "2474510455479584357", "-979503341239819500", "2879236724021058276", "901640783116791589", "-1968701555733090102", "9048825040146306689" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "3527698479739214995", "-9168040692056637587", "4303558786305470137", "-6434168737271011073", "-5987724165431090257", "-6342874047925994892", "-5023544862753808662", "8349405051483082410", "-5427848340450674266", "4416839962565899579" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "122719954361389244782", "-1715029242671728269265", "1046814468132539634758", "1829993007839921437821", "1961163586998902612001", "-1718264963912709868519", "-142775659012283376739", "-227979474167608252692", "-1284136730697963933939", "-1188947702777815289337" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-1540168519262718909498", "1444272924270897021948", "305426275834034841423", "-2023121412074327344395", "356123154623624533101", "1851993497777674677711", "-1548911725567051530406", "-1688091710111258813903", "547593933625451476915", "-1136296230783566148328" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "93977861990199039012", "-725203385003516503766", "-2136208269275109184730", "-1443758672222546024674", "1770464701250178930017", "-1503951609214983767704", "580922121807638712625", "-1715487409849159976347", "-1581681782950208904546", "-345107986036541695760" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-112509459909310806801713", "-318071601726453617114288", "-377093818504933877006627", "-412251199154230320946975", "14121963032451866972322", "127959674785194294868765", "274741358639653860200873", "-10201412639728798865544", "-484633295332684921844249", "325728689994021020089522" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "102768059460000020082850", "-582172705216901696514361", "385654623152916059129826", "-375444952943658610617412", "-559762648769206534338228", "-455225699143700230588676", "414928288856671138989242", "-239573759040735488345533", "-151621506107277003038408", "238527569805762630418752" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "105793826786409638098848324", "-105930031172236153458218556", "54215836825825339383685314", "153999900608311376451254881", "-60781620142763268768239184", "129045242635307916195795557", "142609636641396599505824777", "25997822165486291786720766", "-24843271142157773732898696", "27667985603420108003309363" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "151727563990526262987631020", "-153515663086836104138582144", "34269843807095303273290014", "116745487266501279093435997", "19697558732934548814393639", "116467090539752813149322652", "-49597590215015448554625243", "904912798075267997581911", "-39665579024769081440226797", "-148107902864927713797008511" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-83573886433003161782682974", "-61690975717719632051567161", "87221161283509159769937332", "1424308408986182936520347", "-97659156978716488999314621", "-17069082270469318857712868", "-81196418361561342109282996", "89508773441843844208807904", "144076371744089019894053325", "46698503394106850398118272" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0, 0, 1, 1 ], "DATA": [ "-4870804024939332050762868706", "-7019153028274263100384076399", "-38347116871041435121142912404", "6752651380046831311117204924", "32477977767491441764479499613", "-17206384783392201002575380629", "-27155762594673709466833763677", "-7707471006910257215683844696", "-12836204732773392557927387161", "760807996077187416324370561" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-17307596853554170302945978542", "-29291868274857325327495085418", "39520334649804167187543012083", "-29850376686502020869058128768", "-7769023746550159417970381944", "13728883947958157003762954783", "-32237971097073321264822351426", "5786340858732715593319648571", "16603037987628417795944460761", "23450906837191853879925326964" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-9980201576657945144842882236365", "-7407461373350157855602680465891", "-8849151930540839739287729740462", "-2082450369562757904475450406520", "5688213649545258906102340952545", "8532260187438278666751588130781", "-102084121821929763174388250038", "7229993189261659865189536694418", "6276298854863176977991964120714", "1342743900941457696012670661953" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-2978344615310830117461734259477", "-2947273335992494396678516995826", "-5244820858348685929338162101277", "-2640927544212422127465200234510", "1768434382958061290101667511624", "-3963482937808073490523756242168", "3374307591928127235727040204209", "-3382678138467009217749439169317", "8801701051040866573946377274029", "-9104355797002943521132479841900" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "8221768927807895270484160091896", "-8900000026189067281738645101281", "-8281746127211712061601232308528", "3023240804949676140399638485768", "-4102245491799843728882823944331", "7251501339276578447235263420202", "8751503650699867906907117450087", "-3729506782195610335248190443752", "-10033160781867769412699287534647", "-1250783004758791761527983017975" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-2562167817474569120675956515425419", "-2458571516901048848163704156892937", "-36875395321508089748922725678113", "1960304726616291730330575204501959", "2055550886455178489271580894649114", "-2358720546112845253144262770181075", "1595191993485611135276028859826726", "1566529993850113106475509726619617", "-22596759305941524381058617032547", "-498743035152497361574808016716787" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-47643018102478418616183822213670", "-278640293644733515826236620350821", "-1488199042811674371836519886289180", "-818674521344108691664748392501099", "2556172025333762565075594479138752", "-1334377085385061017822352624180696", "-2000158271833880683895816854711292", "-368140045799924875460317772104607", "-1585782388250425609895544646403971", "1824564753054254497659039848918675" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "479525603135366804733516578850615870", "-495446691382547674716309382436075953", "502812585466542988435075354889645892", "476416535889813426699310670064638917", "-379315281173603247308987567117630647", "607404279943635423469568736054452550", "-629067677540132407295995323971248814", "-62158935415489079301126585948586031", "461986658609980837096262131134918226", "-589275703863823715586136613904443930" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "435625147644630830418626494902722848", "-478332198115914941416962365458575509", "558636662901555390631163102871642769", "355599683788663730974008140088569567", "415606584067363911029530290237838841", "398064603694355571202936926109341982", "-633945976114528848016756385029022841", "457577785385893587465074783208649049", "-616523624972085533886453431515327099", "-248511133054735912122810606062669023" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-57306327392984906276930890383185368223", "72793461807811385650658673069344386369", "-109687531415068367399736525485032846349", "60356410425087146413082196521754844149", "26662652717210939669243284822836719981", "112603258741679461947852680773641769673", "4457422288602224460620455185955042948", "103794515803470385910175949232361042030", "-119185317915451074176129503358012831945", "65467100518198811141064173626789800823" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-68618397724637272526572961589871509577", "7721187761487257785999877840186156513", "5605947584439501391665551914895560442", "68369687165773392157960709162408033710", "-96656920099210957452259257889912964672", "-94072454264796947158929675025216943665", "144069747100296579445022554416200691423", "-87560734517757835226646920956502241626", "-136164322966740604273148909571047647101", "19429304069228502711385669535598831748" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "68390208110339728548757304121957739423", "51149440950819465758183122582606174618", "12265984607323358363395800066794656038", "-150000242236153252396886212934934551728", "122779026475879408492238567997677881221", "131874778752322502170775011108192661329", "165901435173219254996114701417463409790", "-152003734738506573690620489933044183326", "156016528459691385523457640405842540870", "-74618395994120066096952180078019949350" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "23815", "-30659", "-17127", "-19868", "13628", "31416", "18133" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-22400", "-14679", "619", "32351", "-31747", "-7719", "-29774" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "3520430", "-8276056", "-827609", "2153003", "178182", "-4215376", "3779312" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "3689483", "-735347", "-5360898", "-5764362", "4660427", "-7905626", "8154092" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-706829888", "1167156309", "1518125729", "975860423", "995194446", "-1865080408", "-665865487" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-1701907448", "1352934836", "286059016", "-1400717695", "-452649966", "-1922657966", "2070417333" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "1935351572", "765282566", "1987622027", "1741583735", "932400011", "167151325", "-355472668" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-396859212345", "-358864231013", "411692595169", "-32032800170", "79797592838", "494642730187", "482700694650" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "-94529105733", "-481091693177", "317977953065", "427273590013", "-227681357693", "99285197947", "-49749811753" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-117118949406128", "-16338400756989", "121098098144191", "17786074861365", "139116588792903", "-104900603769656", "77460829002909" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-64152968257972", "81807906354014", "-9962726454967", "86559569733926", "-10083720575770", "-14825156695938", "-122499888153949" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-133378720489228", "-59578984792077", "48240930882493", "-81399868956224", "132623478058564", "-17946185246720", "108220579003501" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "27603984537822762", "24285467590617142", "-24610833591141255", "-13707671138044065", "-24079809168036321", "-3426285774015157", "-16661207174485960" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "25341970114930753", "-12578463204570261", "-27319367490414208", "-27862145691252063", "-9374098211284058", "-224593245961164", "34774375621000453" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "1508260868549245163", "6273375209195432213", "-9078384653770555315", "5236211367446183278", "4843815885489957283", "-2504671565165730378", "-8947215402970277018" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "7409921464496531061", "1356029422123982187", "-1364654770344853581", "8837499222236300067", "5132751121490100122", "-502781462678344483", "4467369067622920214" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0 ], "DATA": [ "2154925127820502346293", "-1306492928287846607375", "-278826605081884366316", "882916332012524932211", "1867667626776166611792", "-1257524031015343087488", "994948667658237479955" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 1 ], "DATA": [ "539210603887413500119", "-1351832695016986241156", "-1062904294743508083575", "1450483825798880009652", "-881626741922698995815", "304292780803779267384", "-371319374125766711584" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "243226565466619142907", "-121955990464408141742", "-636816055580076932255", "-513308924855956320428", "-1586486017550068684983", "-197644649227952900350", "1748345750409685925848" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "202061337421568637115802", "185405094041080393854566", "-147903622544387135396853", "207229453925763208622831", "-343183304767942752569844", "-263692037517810930229757", "111266436442025136177952" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "36355222093341524385642", "218768782508355752146729", "-200541398482024855947446", "28474517856269610750246", "-475859723844988539708245", "-300510275404419103333742", "546674288910101331332936" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "3601936927556237989024608", "-3692430919094941422100028", "97748690453108599708927434", "-116661353115321540714375347", "-71879985746676902756048669", "-121080329577202994807755612", "124157596086231452131087011" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-121811982953161783629612240", "1607554249264877581531325", "123072061986545439633145", "-73550837036916705784352874", "-117419921478539398829685153", "-74399538740549801431316551", "-53072519800681711726312500" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "128436901337483555656980766", "113598952940587548025391029", "88411366081005561182297775", "14815457804520490927167142", "75638196408599796922678025", "-108087849560548352593752169", "-11833635007482033712808763" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-6781678961533213012517697687", "33224612367898641792526203105", "19741448595364719290571423993", "34895232778404370472470020141", "12658156696934366509236475832", "-1339988824059695372854332699", "-15240202918093452204022855813" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-23304905447440712184522177464", "38854699585363157300282622119", "25197020277731879349242146247", "37654843997064076575431679106", "-37694332242432992461842973886", "38735096335824029183056699705", "1113712061187242088526216287" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "4872842181239913302717185661993", "8110981189886162420036583272468", "1846154365393107878136252015134", "-7355182167045667821268714569976", "-456342272099675035018815637968", "-4741310371570496786057876108775", "8692654330279187212246590575934" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "789321336877871879752144980182", "-3592030043211469330492425640956", "5543560868004228342173534568596", "-8310481621300119615028360910615", "7240613238493658400891584464926", "6698325613041346941747416888641", "-8117880941116130680718189365519" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "-3557820883496744079071315236925", "7037071399112810484115914176160", "-1767092291265591600640701935680", "-6045295616444022343488779010934", "9259710329844166183361141015059", "-1492876309663740055714983855690", "-8427791476065938140828199154822" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "1158759477908482981738237800163588", "-1173316691740305096863218708950191", "723215252639660437402175676963743", "-1879559334466643497978379903607782", "1983927892969038400024936381013185", "1149553546691385176645909195969356", "1166342561375174956477479521466108" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "626529133445278098702551621416112", "-1408837087511730448581596972023260", "1906905437026049050822161959468707", "-1929693317190147781992957517985541", "-378172107382327021537808059758513", "-2104995076904948347049313544321337", "1294253937974735117873956527399622" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-523256123496869429061522979056899046", "-18365757383114987172474795440552034", "-204316577987191697375229787073710633", "330181965802203666233116582657210837", "-329328597335009708600913451090602124", "203539490982126102130410391161030455", "417738506604065828522825064729683710" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-321000364277288528283109856107655940", "580137014281216496641049610076428712", "-624714144804715993809991357206356781", "-407262969746696369705355949281205801", "481317944191441079472434015349123595", "217536632380375131817558978089536953", "481697272008029766433028053959999813" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-144116647597826623855199496157914248857", "52208802527320772539382939270524160714", "-150911491870745668133846108351239739078", "85442639480033500108059636873587453266", "73892340483729688596932981784301693030", "-21983833769824359378093314118115605654", "155791999129521894360881202697471787485" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-161222642718655133616350033662785590309", "-144401176588064295883792554624730874458", "-152972600928373921945993474844975784915", "-138536917768299990575075582087577539387", "-162776200467637379804062098437779005139", "49042684400063856845441060397203950029", "95767276168967643036870259756918109375" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-39863797120385387096682208969556052403", "82559022270581885556475272207184980799", "117510934395578824951787214497627148540", "-83476646270141960487173091399240016205", "-1758233846624033403425881485773452965", "-135007718267876949864445576181493416395", "-141767357710928769494049512437739952119" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "31322", "-12998", "-6668", "-18050", "20870", "7407", "31517", "30964", "-19025", "-26502" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "1882", "-19708", "26796", "32398", "-3699", "4487", "11540", "13472", "32388", "-24551" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-4766986", "6654828", "-7036686", "-5243986", "6660194", "4688952", "-6565312", "7846449", "-5876806", "-491641" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-3816976", "-5268710", "-1742516", "3856652", "4463747", "4089790", "3013827", "-2806363", "-5993564", "-4334975" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-1939111885", "1089839138", "-423110561", "-238112023", "-1060805567", "1288567695", "820382341", "-1689688269", "966765872", "-1935608267" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-806572859", "2123921205", "-1951507311", "-1152859862", "1529442938", "1573780525", "374401763", "835164738", "41728321", "761957516" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1, 0, 1, 1 ], "DATA": [ "1756123501", "1520929506", "-284683124", "1144653924", "-552942804", "-2120298060", "30126814", "643208321", "1693512950", "1968260177" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-543650305893", "-278464771584", "-373909651582", "549153445885", "-242295101951", "491322040017", "-328958769652", "-24343088729", "379399746035", "-103071495012" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "445831171065", "223984208224", "183764160166", "489898253722", "-352256812260", "132712896758", "-213452847196", "-458421076936", "-337947454298", "-466450812833" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-124879858110290", "135519222186375", "85699747562265", "-85042659127554", "75573515688713", "-58488578730867", "-41028103971674", "50130623698029", "-47534808179200", "-74283087483722" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-108796237752434", "-77267431737175", "51570892952731", "95335199636762", "-120149530213851", "65347251353625", "-20627278920101", "-88198554155288", "-7858756951448", "36719638530326" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "37989750263669", "-7110474133358", "-8142079359472", "-109838866716358", "-109570651345817", "-54071131597599", "-5091765250455", "-88967218959552", "-26610598798817", "135506189432092" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 0, 0, 1, 1 ], "DATA": [ "4927887103256557", "-18754894942080393", "-4912007726659186", "21247575662236694", "2797426944150268", "-6616351412144510", "-11858257489510497", "-23188104958701432", "-8877819571973802", "1148489375738940" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-7825477566817051", "22013917265275897", "7697670787874498", "-9332621858164545", "-2912282090647944", "22112963575670946", "16681702598755920", "-27753683646566454", "6945904955659446", "2442936651087792" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "7675767189136958218", "4376177190988076013", "135914493625679976", "135689732946467188", "-4157597916871123429", "-3202587243297732574", "8570927145651867846", "966150880672931514", "3340433868757095215", "5044440160311120876" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "5133284943736156946", "4845520967720236486", "3808412010600114265", "2390070001401807490", "-2075341331489173819", "7269598711719193440", "6639116030361989708", "3990965590531735058", "6049354888289615042", "4801937515511789680" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "1064355765009099408700", "-321943498165958622892", "1201671142156144011576", "1520237466996047474259", "-58924034610474786199", "1188600272040920538189", "1973243532799526539431", "-1589754043423107159161", "1069084636593819465866", "-2225914539586890120093" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "612790270140756043932", "632096876907248352897", "-614445606437892415535", "-1538574921109377688884", "686539122267683104890", "-176896054618733272731", "2112326625855675905778", "1807700217188087573935", "2250554304073443683530", "2138601425060065078780" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-2178158937595239087007", "2148658261965542925597", "-1786860273154704658204", "-2284942685968830599534", "1135179943332399887722", "-1850089368721794824531", "-2107516150972307207136", "-898833291339509074760", "-1560291136165671096302", "635830273331647493827" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "545257608208761234784401", "-478027394265681567220331", "23160102878411060468572", "169851178877058380224552", "-125063444295476267854414", "-511661104901652851551160", "-211091539176940271013432", "-260892604441446245832984", "334071806187190766343296", "225652640236774742401805" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-125435440033823628710650", "134873701855643950966740", "-327244905598551933045573", "392112435242139096124146", "-392770190944746034104800", "-444456098660755532238056", "1948570436020667687581", "334511617766662717840293", "459630175785578568834194", "295768689726958240598895" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "110764581191900446903790037", "-142251040052070922679906758", "4521074766144521546724589", "-77954834062755231010987947", "132206695009844374949071160", "-115611166334940287023117326", "65706530528500814014886945", "132926781394523064114813219", "56946888735676586779415707", "44978068193037238045912896" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "79398695417455848981455196", "41296350950824030267042588", "-44890473232271813368854431", "-93845863653461695811544157", "-87673990780796119057141077", "-79728712608821225741329213", "133650017782788394397592147", "-113944397482899885204627134", "8147591003946322262360118", "-88347016701263595509923031" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-97315706287462853410250215", "-19297884994476314092921079", "-138156245563462225217055556", "140082648536915847982187381", "-116717372906786739105728127", "74725068650981797124372774", "115115819529181418953999340", "-104692994218324284120901952", "-9339214268717197582310194", "34586415164615511096229871" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "30452540938400888864313250312", "-2802879378440933700326909109", "-26896580846052300048510748778", "13956100052440828740572023117", "35092158772914033116097802591", "33847590995690391053397721124", "-6872021573016953238017942938", "9337504336202853974096101152", "-8139564364961519011852381841", "33614459434990180894524889482" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "37905470020515740655644953427", "15095043730643435565566661915", "-16054339514326668180752045709", "20206326378427568201157613061", "36409538996611392332800791135", "3986926868645572538910763024", "3619435534252705282359941231", "-28939638180493501947435665546", "27752306518932422634994807389", "5299661533164103665947907747" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0, 0, 1, 1 ], "DATA": [ "6105089286518535143859193272997", "-1558013028566094088457684756095", "267447018904579079104631493606", "8716274064389440981111441588450", "-8523502283934172783824252808972", "7211587162269446695523784988664", "7349202932851786545780603930347", "6835046636946129018720692573076", "-8333744634109651799888167347108", "-2544747380910102910856810717389" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-6352412655736206068742730200924", "-2123175620394690676305322433458", "-8836382719876284105833993896413", "5250128511200984854060353093256", "10117785086990984372555197700371", "-7553019007793539065221400188849", "487332886938332402116864111164", "-5531527231691294946689230570557", "2033195130458534180764526511590", "8418380572809691654984623098184" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-50075375212232172188420411196", "4809924043195256813690822020596", "1275374378747394211129459178045", "-5194775733165378826646329523658", "-8899088366764662523040904888258", "-7065161673811581445863416064724", "3448352081115495266715396247033", "-772423708139560818203910008799", "-3716422423592581053204772584415", "-7073723704623872354556609478292" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "1662201874412358678818367544757313", "767632843116354510741172561523193", "-2574598778497120728925932302019195", "822408943008991309024189274780663", "1442207677524639837617120203090781", "-2329938247319978136011213385811089", "-844653930290095052875421889147387", "597209484634644162562668851350801", "-1653776464297020640113550379942905", "1712355361051968314047230183856759" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-934154790921317246804917189844883", "2011996172621693547547251717415199", "-524198925581653076023930890792205", "1460963164679956289335063949063860", "916547324738412638652836279760881", "-2181480489373236251201848006843613", "2227194053359186942441098677400067", "-2070283524814069666316978457528190", "-2557886824011973362420531382315055", "2359841497596077952056722100997103" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-244737191158099062140625546830357355", "-558098781397099000894773898031616911", "289872866815248801825838878888401974", "92794478567836555966241203902901710", "244269753519144613414831030049656050", "488344310874378240044622887144568360", "251880555776188717959293063210533656", "650424241144052628672668970511093431", "-490149649831311456535113419187895360", "587815026061852779759502140522870365" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "372180445289567816296974210709998262", "649322101444650143089117190099040098", "-261315503489678515225280175375928660", "-522083930998546622308238829319416419", "142235256333754647127831983081551205", "-166535174840652571060843789537974209", "-466105320259826015444363036165976483", "385149522321114928821540160206410292", "-41443525693038526295999228480843805", "-313051347741978666120077422667886094" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-167439439468877308797697749268322963993", "-28242537581789671178678277260561674281", "130587384617519710400745151774880402494", "31589263496961519660807111701435015251", "13326683789176912862324807380879018213", "69851378199071338355039736806394653047", "59447861241933516870558227322235615556", "125427854769440091466924747763902774485", "113841414377172020356030816155031973980", "162459891215489371457804173662865844603" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-9113228729168889224553302821519848527", "147906368719665671855384750115565399639", "142408267705249397493244319994103808727", "-158173827712728250139318830913214547586", "90457292148533516388688156622880399300", "-28762317200208469682474790209657754600", "-147267994027501072294873134903092981782", "73349578944467970931133887330177437829", "-109559717261425791283100898451546302168", "118124046303323736396291668012601665112" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-92141949280218575485253751932496543626", "138068266350197970395619500727113936796", "-57057880351028370035446451706682228912", "140046511005251583337314294112597416965", "5313615659947632571000091701353480420", "163618210678848808873963839458536967363", "-83020522347645062930841470958792915560", "103388812228278752501831367543501892798", "105919923091739023980005477488419598090", "-15642676827816127683186187091061039771" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "1173", "-25049", "26985", "-3279", "-32375", "11902", "31640" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "23711", "10618", "1182", "26210", "-29042", "-26592", "-32390" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-3383836", "-330331", "7170648", "5645424", "5519488", "5932048", "-2721797" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-5680425", "5530032", "8360972", "-1579435", "1281443", "4742079", "-5190818" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "791809523", "-1726831311", "-306766525", "1510344688", "1557526531", "-711666607", "1285681573" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-1588122426", "-1825651554", "-854908281", "-1682899390", "-386907059", "1124699281", "76032732" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "21696611", "1114662058", "-398379994", "-37751028", "-1858929126", "-1571912144", "1357766028" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-207926907151", "-108845084422", "483890046680", "32829509085", "276444998455", "254643057519", "162285997419" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "51727593810", "-334061936348", "30150675407", "-331201335155", "-501714900752", "122994273993", "-313427617689" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-38665729792403", "113150504075646", "-13283184519777", "-122268737353293", "7114949849358", "-76982898428076", "63703755155967" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-631968410916", "-129868532669274", "-138633149135123", "90053223046351", "130107621630474", "-126528386107052", "39830786351675" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-36266273353597", "107910261195518", "57616988744255", "44429564088387", "-52991708070673", "18856755200112", "-88910410722229" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-10733019451407946", "-4637031550251259", "29038597614687455", "-2255766756551662", "-6357871065260787", "31141018628250396", "20316885381831172" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "32276626527716918", "1143035765853136", "-35727015302634840", "-26484971152317146", "-32859967854440443", "-32563301944872556", "1238987785486865" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-8755802922599672235", "-7396181851925542211", "6711186749545114591", "5590049579316174533", "-3165569916335790183", "-316326350454384555", "4929314904948241377" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-397606177624923608", "-8426436663324202280", "4326137699230271144", "6586316848218801011", "-8091380893667926501", "-893221278247799034", "3462259882527377933" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-123690421269484703760", "-1267016901460774922921", "299161792651827136146", "-1604897520961911844593", "2037882344461012186736", "1195943811863661819614", "1207132107388937164804" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "2101648800869166208330", "2274724666892686998217", "-391260607449403060721", "-539342914187070082276", "-1440115909019570656078", "1971609407978951853616", "-1123962484080572650793" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "1226504478266363310149", "-1336799843493576778914", "-1930283161332942670068", "-1331434065508788145751", "-2246411720300345670491", "407117263123141212856", "1931974044378992306595" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-361522891823295867238478", "523641781996221269629669", "493463729381734013522117", "132573578585472506680774", "94595327070388358792040", "-475960519364994802696270", "-400496792855539234277056" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 0 ], "DATA": [ "-83327604823851708214213", "-21750678079158391741287", "303079990127928613453200", "221033538070216052397251", "349922191060743894146041", "373614674984738281954995", "35960928414177633291510" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-35263044120328302148125905", "119902294752181475961380716", "-143462823401917067325450150", "76224465900409165961186326", "129425531295697896902175651", "71961437570078598659959749", "-50542941964441222281681861" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "21273851514521946688339521", "-114682664467937571727316318", "-87665586498201782141883102", "-37148627339581480801973740", "148715674812065621394995997", "-11949810986329503823448971", "19902828440769121844807521" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-82305650498237902006296677", "13405144161056810438291528", "-55538748559515558770298655", "47675008502605479352214681", "144416507011350857414309005", "29779427576479727347932773", "76124247118585246765896254" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "5201245639226497436237918218", "32394407131966840913830741879", "17275077614546392015968733648", "19680639349359359322458447337", "27317116065055333169824937849", "-12746107751769296292286176888", "24689167759685843386070714338" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "10674526914216998427441154557", "-34360582954885045536029895204", "-30414741650041537218707792172", "-36100155712815379900420808706", "-9135627534401552379066184552", "24739179676383158523222359862", "36247531023134689615999840678" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "7728506401956815228532038088763", "-2837122511678749937658222392985", "-5052751030412737719634091683608", "6267848105390862826653603709957", "-7256424895588253047138206019502", "-1465562444156137793663621199168", "4747031647779859427140201067878" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-4615761627649385445936725928202", "-10039625391960422316333169130120", "-5864157091799713513469653318184", "2127816811402154227868729483435", "3887900797372126787242346200516", "-1886685381966172685286274273546", "2174897160271324780574975683648" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-8147543028570098325830702033212", "-8166347530503241567728663732295", "-2400152589461801277732655340968", "5816370023067145183233116808816", "4507165196511235167888642848780", "2286027903888132217006101817455", "-2479439547905569438663575023840" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "1828555317791876329482620856869918", "-1789881062592639492432854851944912", "1997518813798357124855744528406909", "1983001417735981157064295426331782", "-469859785106586780643284066769335", "-299069863301987956957383980507514", "1236379623573944194394212606448257" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "888629009780912538325980386511617", "27717296778921441430770399511331", "-1037544761462996724025231718395229", "-2569726393180105871878430433407165", "1050434002808238746161893184408066", "1631428512670349250299644326318484", "-2300221672111018999696902542660406" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "117708577531638404447859755168996138", "499980650229427236421511725860459336", "-11095778332866038210737561230496935", "-545351732363151371037809309111972618", "-247593431450103821518249361274651770", "-237793299013077356563098927682674357", "390680910028520326666704921002649497" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "451974522107762371225049594944291931", "-105576308356387969354658158113358895", "131444584597404940972776636357533837", "511081773875049253533205440188898072", "-152320758908721137229679446546078860", "440168332116421922278190781376074082", "422466258163678281216225964929359699" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-157339119467114553152239726866392495", "116897764313121017716807765578012098753", "74820773589003012395258853673597787962", "38832900549006200655588104913719049413", "31809184412064166455927543375666243747", "-26330135739473810705643960803773520617", "-118932439894932248602393041154357721181" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-25109297380056962038867001208740287232", "-41675596142284332174303529214392893663", "5630763719989167974840000823279633369", "160052832392163305375298352935547628724", "4354500338288930103695848603658025851", "-79240341020657408457197673660870174461", "-105142963009963436353607279364134642269" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "98982080008354791864222553245931141612", "-14255476744570986458215294770192518639", "75075249011469661237617297182263259411", "9960334250615541536807663596098652831", "77610568733923384998121313535231559129", "-124628932104686355631960480356638386284", "-54909581662559425139035632209125077908" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "3832", "1722", "16659", "30179", "-24098", "-17484", "-18903", "26194", "-29015", "-22705" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-28693", "-22191", "-31428", "3880", "2222", "29307", "-1198", "-17255", "-8873", "-16187" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-3944372", "-7965251", "776136", "7478406", "479063", "-5101226", "5809084", "581754", "3714208", "-78676" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-2197720", "-5978443", "6040915", "1115058", "4078042", "256907", "93983", "5378385", "-2933287", "7437051" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "461104817", "-336460791", "-1866209508", "981614060", "-496512504", "-1842967640", "2028811575", "-108083496", "648817093", "-911531546" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "1855062455", "-1848423958", "-417919719", "-1550324221", "-226454251", "1897143013", "-1816207352", "-1890930858", "-1050656588", "1380332427" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-125228080", "147425902", "-1803257123", "1133430594", "1104179937", "-1155651362", "930863958", "-1372345558", "204045790", "325493759" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-43969925332", "-423782533903", "410599621485", "-384348480285", "410563996751", "420331896854", "-451260071343", "-239682713334", "258658798561", "-490359578846" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-117878031314", "160469154039", "265139111256", "479577383616", "-359749140482", "223868593231", "-322904355205", "350722353309", "408233732665", "297350052059" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-11402130119583", "115585152210399", "109875950346614", "-26937162370986", "81368062928951", "-101279737882071", "-79021567150608", "33691525873326", "94018510142332", "110175572468354" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-10950828627484", "-45009616517875", "-33740390224925", "14272990246117", "-77929003338940", "-102300441781475", "-88070246222892", "46505217555818", "33852556004566", "88581112696689" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 0, 1, 1 ], "DATA": [ "120432203274419", "-15955787808560", "34948631108647", "-15994595995724", "-27803388562197", "-63066930361908", "-3386645943750", "41101017052909", "46443568439036", "-130299379020281" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-31001354878037938", "-760641451157916", "-4527935698981333", "20422875389727418", "-25561171622510273", "6859613823629514", "-31967296838595675", "-26772749609822819", "35113008647366866", "-21247928407771622" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "16605450096694934", "2713804935498894", "3748040022825592", "-35884925413042953", "-28842895601915851", "5002209115350832", "3529334705060475", "16112399859517651", "26710552206102001", "33271555054113480" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-6948678853416573641", "8088096296940258606", "1788919402499424939", "-9045098789632663964", "923217667787488454", "2113233926511753642", "7674437743867941425", "-5483523786006335142", "-4901566326841744867", "-2159924707245153950" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "7483952831904713529", "3981681156262502150", "1574702197696520191", "30404321841739229", "-16526697791658885", "-5887792884766481411", "6480092953553123685", "7930432301453660255", "-950334699167714593", "8999305224741512342" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "2275373047011451096072", "1663741816301024567004", "-1866492858541257742972", "-2109145933811908015055", "205702837245156733125", "1635077412214287742642", "685205307788793936320", "1889860958451267333814", "-1166602441424869654187", "745504509663417227326" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "1798836380972328231764", "-1406223119331403767737", "1660047033970478613098", "1778052602148636827065", "2250316159389610073455", "-1298324302681600099899", "1334078532892003231310", "729325161456818198272", "191058750304629973982", "51010503554643728898" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "1847340334856600128313", "638235693099620437223", "-478038004560773338092", "-1391998540736358145230", "-626383328490987256762", "-2352574400149631230455", "-1938941449061847831383", "-526607055338931858789", "1981655780781527447679", "1545396686837274666506" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-312442107296555228736666", "356012445393395243219201", "505759552363745699184467", "-468676292296122067383179", "144650032031703545186687", "-26329362228718986332534", "23980362387843468011798", "-365244050437485466278021", "-243995662429130162331158", "-204320605551784536257214" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-45575166340192063213221", "-573081770149787292165894", "-265165687156774270417407", "142789058928229337895172", "191301704388489595602183", "-526986468133892967767446", "477245941154730186128251", "85837727744584298073480", "124665454939314352568168", "-538236817734442648171792" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 0, 0, 1, 1 ], "DATA": [ "-31883334627754186218955050", "-132647558483145987490002060", "152621615451762950923746193", "-123974549416047643124802713", "-16114612038666942583062670", "-64534320970354404240221340", "-52121728535745783604990414", "-94488926190663283007722749", "-142818466323905379126560097", "-107911476720388673547687265" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "85894584079322001591239437", "15214409976212802853836897", "-38019531841082459805788664", "37616670829337571764768074", "97369332259197095859847091", "73772357245924419843255015", "-30815777011884938279345071", "-48639604753962660078735675", "115468028917195898397228106", "127926946301889108741445954" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "133666088035672066583524552", "-130261035231129416075204296", "126288325871568137717986524", "-86845343621023472085945052", "107973761265249358331477587", "-129772336942492423381739079", "107590361807184920300114694", "-15925934067100476069944577", "40349773922798370876587602", "10237290409613547079342522" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "23802936985404384833087799850", "1891175066573372917579879025", "-3414847211423679295209241879", "26381566039446157654614814849", "-27987571844346932586533728809", "19734355232998847428072102124", "18813566938676176789254455011", "-36503995951399515178819792357", "-14530867618962366745963282995", "-8849346604262124212079590667" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "5181898859266891446958312760", "35107669831244838496494366387", "-14410695362339474219725640659", "-11591179713532133832522279888", "1322035740173050242109098628", "-31330745687123561728243165717", "26792385853476620307255105569", "-239671734221072238217774753", "19102158857542933762328806730", "22327891859950141710268230741" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 0, 1, 0 ], "DATA": [ "7218784186398202012713467930238", "-3350235717963492612747364671098", "6231440398343118743080139900050", "3480339439152451974013676380123", "1802222928179100234342084269172", "-6158375179378131392944011341388", "-3504666018658185926339154570280", "-8475441398809634118910111467555", "-3093801338563751188575901637318", "-5478326702705048089627298988765" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-7070255220529922623499452044373", "-6392207880846944173023026301966", "8940482032109394222736828154202", "-5046383856173347902770746632199", "-8627626268162429998930354857590", "2158989160600634779705496438004", "-9013792196143998093187489447006", "3476375415358910964701042924829", "10085089756175313037187138824355", "9740053811517652352505683816287" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-3787663618609735534843807278261", "-10131105569536599796884449801691", "503460830171569790542890271138", "3434572231596005739225905817721", "-8956835466725390809094014178560", "9973500372789618871294599932426", "-3436060305237781453095163214550", "8450001797448541892298668439883", "3361188979149779406972984767155", "-4211051396445129112263473603788" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-1400604743906584917351131252499625", "-1478062509467749242110369849940624", "-1939775116070969545684922458204716", "-1226594030808115703890602534185886", "41411606629976216575859515000588", "-715726075155862393171728350610649", "-1104837525141739703305043777788269", "-33170364036380943653659197533160", "1222412368140077471220168238614012", "-1553672963167499514929514167750075" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0, 1, 1, 1 ], "DATA": [ "414812444544593930314879937191796", "1065092968521305037826601157806551", "-1284545736853571361465316035084177", "1913928682493656194884471773672006", "2437413883363914925409924254101191", "-63504718310075110440759765897849", "1142014044009551494450713989884540", "-467473985179578855121650728580161", "-1958496203046466422013468466142816", "-228498365668817309414956102721486" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-165107146600836184242508744416846410", "-173159599341289250114641790452112880", "519587843813546128711723035455451318", "556002998269748818044894693715246655", "340836458025000843041754711402673966", "107273561582467343086931264151931276", "198750867607657619623292872099343193", "177053814334723551673700131599196681", "12660542230757668534285774032322329", "-571464786368411573231284232367569493" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "162179090271636955997303945925436088", "-313756832601974313228748371796894029", "-479290633805655842621397357725946736", "-511500927368428520551421482817201088", "275242443136997885046391431699367254", "-577640700565811102564722509570839516", "-603313958861080101850976268791504285", "-4642553372325527067527268751318523", "585149029480379128228533930021017452", "-447624843136876027041393586846771215" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "41501085361451418692451075430330699392", "14066756191036297756834058689490291695", "-164566469205370280337196747799862639279", "97007518905806845925433851317020841140", "157835227951254852214602652284268481160", "25273736966363553104566178634088585028", "-150537488010280684140991686519150494549", "-3078910762200766609996313414922029941", "120706844608861526819394623973916082740", "77402899003013119828768311588427778798" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "17155049489457881084030524101061900599", "55722466817055128162883412356150704414", "160092374234431079498351920387290796423", "-81606519127017628665631973004297421491", "89375409744877313802670851685928047535", "-65994081055910568627784130329323864802", "-120630990114675994405665160336455407373", "-137414651139106149921771512358288667134", "121174892531857852483583596440818324131", "-165550685491777099186772462955579166433" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-121678364556055223485335845024798563730", "67906221171052313441526344645309169424", "135318293234875105755625723591118174647", "112856765176163913169894958961682862611", "-119771416342600648005955621630454841004", "-163903534238729373402472686918104270530", "161195965019384544589699058515114347869", "-116587313658056174021511449216936299530", "64724954802600766569366551938003781249", "-93125379513310589407008602216272655289" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-24289", "10573", "18399", "8295", "-20661", "1262", "5316" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1 ], "DATA": [ "-18813", "1196", "14506", "27699", "17260", "12672", "-26712" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "1816906", "1950630", "869435", "-4057560", "4225264", "1863554", "514068" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "5766833", "-4566718", "6185712", "-78858", "-4404053", "-1932082", "677179" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "383051226", "-485372892", "-903491488", "1167906715", "882058907", "-1145532790", "1869671269" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-1968019064", "-1799227215", "-485104210", "1368798640", "-1217008177", "-1227552350", "1050282791" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0 ], "DATA": [ "1331776197", "-1523716584", "-280980420", "-199394097", "220377273", "993543643", "152558613" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "383358476044", "-549418593050", "-95127854833", "233604693010", "241360165929", "-244114921916", "500668910648" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-53958362657", "294585258987", "-297555344274", "208136066113", "306802849873", "-56933397445", "356355330533" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-5931341008015", "-85194361749904", "113496344354023", "110910986479677", "99037670275407", "-29955950246308", "85080062899793" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-135973903302549", "106511990735629", "-68491062797510", "-67355500519171", "-90256403179030", "-67513824484015", "-117243873668118" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-100006712547242", "-834254212295", "123327381870828", "-47227724753173", "133999742949746", "-18734787485868", "114072994453650" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "26713924791564839", "8130537069775545", "-12577784584283418", "-14275066705387698", "-13753875085512229", "25629684304719161", "14775029745555656" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-11254714743780398", "10473563621744326", "33848039724296273", "18968181007120441", "23801217524644040", "-26532014979132953", "-27043991311421046" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-4338512873691242591", "-2471396968750798701", "3382716270394008960", "-5536237505914333541", "-5670361551544238026", "3775163668383883090", "-6190677986420548067" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-6811298520974526893", "5349469892177723131", "-5310390011268146908", "-6394607027886038057", "-8840279685184811878", "-866886887884920562", "9106364263997684727" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-2273577051086748127343", "2421110092972784882", "1009841336246568968530", "-945047452043730572283", "1410134830331293109638", "-875429235306554264942", "2114146046206025795606" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "2175914150359547376158", "-1748223628477889384538", "2082301040631284414221", "1223990770664639894611", "-1827289874583926966422", "2188372252873287105874", "949660314176845084876" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "756120269623394822481", "1401758245452765618339", "893699234552132134074", "-1016562455395417533129", "982560837349843281349", "1535949657790955542784", "339935115964234018700" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "160187131709749816068707", "-445406864840389796795963", "-364502369523172897909179", "-483741481440179619808869", "-363648688131558141030019", "-317255639295947426340261", "76880251422851734014297" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "83513645025310150339154", "-17033888719894118308423", "139753445403319042050931", "-311922677215511648704490", "6614071525705894990176", "-347208526123811355958702", "-331153931143899968239074" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "66937915462785606409890218", "18402929024834529807256440", "-93539116539481832132130597", "129533946990250575778822929", "-35405911050564377669039868", "-55753625842174688440739221", "73274477544376298222515222" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "102320970151175129799043835", "101371657730044294428907301", "30763947066697758866083944", "125632427074532822811696240", "67850157625702978153575284", "-122847530486522138663035099", "-51630753075526948960226330" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-5570796149032603732772443", "112962510499227217269980469", "153243861290764227222710433", "53437571104225666803388890", "61392106787538463166584259", "117240447134205597906098644", "60147122185044384664950672" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "25440681132059718117365230086", "36918164020123084473254266745", "21550823988603061803770774218", "23314393410588498837026340595", "6838667569178219678527118012", "30164610006780796397257146070", "3856966332228692365975619319" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "5081531030911632494676413206", "-28718432056933259489318591062", "-2050881926509555330776684358", "-20284056359541046744053496159", "27742784349335929086470876577", "-23732359260540767132988499358", "5634504669489702253559754357" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-3834438486720541018898060342396", "757405556339035170443861964454", "-8580423491565526971567650107331", "-8945834070161348883399473301073", "-5974403419273083538716640073054", "-5722317395097101795951011911463", "-5741054963623745695692184888504" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-8177592807298787544136200201401", "-8737768036785410829204705458137", "553812896188979268090905076767", "-4021415115048931628518326506347", "3924889803517574116437664136082", "-8427629394384391196070489759500", "4826578044967295536262713612905" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "7431086482852698405039291580358", "-9383454246382299183181940802026", "110780578216389910057065519226", "2997597313220317309864866164017", "-8403711664152768877667363976799", "9484934137736292785001782174450", "-1992550614924900659257438994754" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "2373418612554887808061895526341322", "-875917552657361505492351804147083", "-1898458663708254096173410834725291", "290657266522040208254177101419000", "1934693067868945395803972374030326", "2035880975736537411226150165204569", "-295118642138847608101651879233223" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0 ], "DATA": [ "1104518889416566749000188799690936", "2455684185543033692597997273850286", "82712851079703819661084329753293", "-410022871087898657272365213961056", "820614625160173093789799139004878", "-1777050003863809764253453475090598", "-679519739224009400590313815819330" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-304159418192963037178769623546586814", "-128686367459246117841134563726117419", "-168852189389901620284754066083296623", "-368361507059344027859608313034640451", "406153616128697783106760476034969080", "108280339193448166575810552660499327", "-272917413831425220737435317129240149" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-521853357960558893783030665586434328", "-343385983135307684129023060457977073", "-506847810949852766012501602241167446", "-320653206306289485921441154060647458", "-321419500062747985044408215510498559", "360059077100522851702855678971772481", "-18193071889338165594572934630198306" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "135198259031897173424041180828687717318", "166925950940460213304747134486553784378", "91770708378464964972025279535217192484", "-74034082172265236168468281957827129715", "-162271236574132326749987187978014522711", "-149970538812246904199283660851953012127", "-33876209446467800177023509622790899587" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-61149961594095579505494563784365845076", "55447541107548109725109916957958410011", "-52782699847581819420368482823281766808", "-61428023230847673671328947277950650863", "-77167370229187325704618575761108559915", "120671975447784265485882220443592967003", "-31941156027929177364936983495588921100" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-46517984021030779553174610760370691361", "113342560476092078423544510209910511761", "-48453285416719025136328005628829738351", "-156382820183969337136212148728801068593", "92013865013427171673489829654013623118", "-149825413114872556503600279014768710017", "-41842153899195246897218693785873371052" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "22560", "893", "13712", "26425", "-31443", "13197", "-2033", "-5158", "4272", "-10000" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "7568", "5297", "30894", "17857", "-2276", "-23490", "-21021", "-25476", "22182", "29801" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-6043434", "-7233989", "-4760386", "1954881", "6262402", "-5432480", "8247099", "8104542", "7559904", "-1468008" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 0, 0, 0 ], "DATA": [ "-6009851", "-3463144", "1902131", "-6065871", "5322522", "-5281546", "-3949430", "3706705", "-1404042", "3601699" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "865502134", "1417082065", "-629664388", "-807386741", "-932260324", "-2100594554", "1317544706", "-1659287662", "1747762053", "-396035137" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "2009773453", "1452039055", "275687455", "1229090980", "1647739999", "-2068020074", "1116868483", "-925674893", "-276647664", "-2124051141" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-652548523", "-734515103", "288855150", "2003714209", "-2050720739", "-439439213", "-597448192", "1032305015", "1396503618", "1112795070" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "279530206260", "-37787584606", "-469700465159", "505547261276", "302063628730", "232557570106", "-400244326424", "-500191912404", "25916715111", "434497276265" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-350338719945", "82553936444", "-303574370324", "243609147006", "-406918086752", "-88984556473", "414777054250", "-65665371841", "-214549984551", "-160387504734" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 0, 0, 0 ], "DATA": [ "111115937773490", "93917065955228", "-96942832219834", "-33245985521466", "-14808380437022", "-4337750370781", "135538464766646", "-89018159354352", "107684460266199", "54644567775645" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-4281375143734", "-90555175252799", "111437459291494", "131211416029101", "121367162942964", "41802077584397", "-15870395962588", "88293896383396", "5103371203595", "872421043658" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-27344466164330", "-64277006280629", "-115200530257737", "-96185525872446", "-362102981191", "41076176727009", "97608038546518", "-21909959043825", "25684146129386", "31887816743174" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "15805681068407129", "-31632108725366096", "-28430420565549892", "-5685098119469251", "-19359673914220516", "34927075452235781", "3646450258861011", "23186600653867991", "23870113232131846", "24001450908961341" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-22903183193209387", "-4042683237615665", "-2730673747391472", "26722058029955156", "6157099985124238", "20691656929186432", "23541903929043982", "30481714773589336", "33354780396992065", "5937390849575047" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "7111976505000124157", "2513743790789558989", "-5854964827243221874", "1781418857572271171", "117740201774951719", "5464574628727265159", "-5417016541752216420", "9177082172732916663", "160231964650370102", "5198235504762424898" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-6959774822262382900", "-5918004911967555855", "-8785219904076781827", "761518306349693777", "-3727781912512154347", "-7347378201324932678", "-7407703676847128120", "4192766707565496640", "1019303794324002221", "7284694249235670661" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-518706270599601321284", "48296813336873849945", "-757973019222940359901", "475279089186345082149", "-1857051161967159641567", "-1773449378603505991898", "-214240478131630835049", "-286969346226347726119", "-1377522753231516069488", "1534327277443683237083" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "1261639334685978762634", "-863251376487397515329", "744768807612401835863", "431090683079445762349", "-1430008898903453262513", "114432723295067239367", "-1659415137482284952255", "1956229533554192977542", "-1285276599317909495561", "-681033070942915028623" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "701567718623448740657", "-770039146384883281781", "569393332403074078663", "-1154152301490888490101", "1384935762862036151746", "1292652059491738064552", "-1901055548660828668152", "-511050274918474237607", "1514545794119297935503", "-1582702036644800787512" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "525113524901831930800014", "595544469553401116745247", "531694395790167129406901", "239795230404718942193473", "47476260565690957269128", "546393256156863431003393", "-61262268239682896208557", "401221833939464710528829", "-300134644671240853134013", "161147446687616707600843" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 0, 1, 1 ], "DATA": [ "219138432946234236649127", "-306826762775104215875477", "-122309794531015699303879", "-484390629051561859282314", "-484657707904256453204167", "-64172125540895226776153", "-142182935337162713560713", "-425794894066631241856692", "519332704943479362574611", "-39607393270604766751662" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "117923250549289739547762878", "-12796961197805647943116125", "116897654782120455942318786", "13956190831994724793021311", "130567329542252884379704341", "-151472228305034689251068623", "117154604904044151278566211", "-124756046758763955771939571", "27553563965416818601126828", "-6385245309521757072601303" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "19686173839910969340904667", "67019339050451913372960344", "-911760385169067625107436", "-122602505415495630018637800", "31593555949045884898723169", "77102155795733768673367210", "-88704824409118698981385569", "-42307172980137248443961447", "-4446290230048200090988726", "74356647474998795935207344" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-133795720031537585830253848", "-78089793241413791675804237", "-146996109993430013479432790", "23768136773568272325587897", "-106683272894876981717341824", "-80841936515652353087463629", "42569237872224343987105320", "49071873620207544414478264", "-4630034012596569492380443", "-10118444529078594226753741" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-24951441861790452223123296595", "11386091573469723230249468441", "357584100357389438039201862", "-33986416720044219140319583496", "23627532621110890373222649794", "10017504927354356464772980155", "19362269204948111616152932551", "22750490848378999604678136199", "-9083598083282026421514694140", "-38727975416599385546239458974" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 0, 0, 1 ], "DATA": [ "12488280666875253249236254683", "33422133051718018037208075117", "-33776414240503546307124956974", "-21484107419134338520121082452", "23100427643229869917783680858", "-1650226751931233622609376911", "16906427077642537130376239920", "-10289660454830607858626682520", "-19484893295053214460865724970", "-35952156995798958657792954473" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-4735298964100410577985543000395", "-530020970016182803114777861279", "-1131349626390530283723194772589", "4317710074952661856482753164067", "4095402110127114773697774757004", "-5294319980949349221369104353569", "-3048927073292851696777488488702", "7634763577542382376327040341451", "3115557643024666160471563838459", "2926203117946390190222302068793" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-1864945236373483656597257239959", "405034584047843548124421910439", "-3218977334120915262993640819384", "-7142961612298989061130444521419", "9307783629655608942945035515333", "9723234906857782536374491021270", "7650370082443858700585254978477", "-9310868784454660367988144252963", "7465773199661339364694190806591", "8097845381146606888985497903233" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "2675669436671053895099186040413", "6895644571078189440948660962592", "4090458704020642720758701259110", "3437150126960216898595459196390", "-9705511727116548664311748297324", "-9305522884159558955104155253458", "-1180186469868495251804854919531", "-9796644010646070627213151591605", "7106015483167238861336648112959", "3331018800384614291320519803020" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "1148597479167394014443766490953345", "2591895812533789822937308818442541", "-326160310667933246866417794387851", "-790306080625502821695639512319184", "1545324828980338934424171461753128", "793479953457908571858622935305894", "2257543789427477727739601853041034", "2444208137442914637553318787476074", "1042890315365379193825287662234332", "-1210287719815272207093258458478606" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "-582733012088132875430838862529115", "-660250931409288922636705489970214", "2138527182118079262740549727787747", "-2553900648599237329560900255287994", "-818610871820914906670478944208743", "-236741329036188976369182885941428", "2127636973795770254193330141423807", "-431302383365507717941243837426762", "1669665166902924237503748195025488", "2520650475436221136120486524455387" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "437759108051476501968471376399124534", "-243331297098327608976864926703591085", "-312583708013495348602181422788137336", "22789177262884761633952302550572526", "-120348891169104228873609073292813076", "-659626114206394610798043021957841914", "70595700062132590775654277374702719", "3451720916899085830004332720776845", "-34955038635481392379717309447778322", "611134933947728200993481817988506418" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "20428164720806545350579904191767053", "-99366399181425663956407652863667479", "-35862534912239206067332175346609413", "163496968286287738613339640023311242", "-591381118335160981639939232133613024", "254858944488772779912509622495446951", "-460577318473676675239602831217222871", "-616355768017271613012286794164991144", "569247714773488133953930390229542704", "202465807813752300281405751766207660" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "50315610630704356585322877730374837077", "-83098067869442334415610073682313947471", "161365526209509778448649648555742998425", "-34910129007785397951451736271574830290", "-144028959288053184684355803368266298023", "-83817363552302335547213001246996745217", "66067533735734112278087882548638909770", "120913070546179844966164010066001238079", "11384422330934700411529312190354569876", "24772640548850140487291953136913828116" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-33028393081806170731403400067321655079", "-11463174497738743284114180047596875486", "32254998885085389327900118303255482169", "-143483787323458716777165205934466305713", "-112857257202882526765530368259175091713", "162300394541802787040467843446805562970", "129149449533069183017216694533759537096", "-64890619893840122581192195171585201603", "92544985698450754032974399211134668880", "-54842712948210953012984037318158437034" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "132478357453720312304052372488893354300", "-32685327131103251164646706295319990901", "137916669492333509181609802124805628257", "87478174466828645335209662043146117532", "-61373497372439972480430645755553315203", "5894673095536466472878429843436835629", "-11619138682611129587257806902056620796", "154151469754446633748882542018886132033", "133733328652939407559328194516170213391", "-126168609308185025841853092223452728320" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "29705", "-26987", "583", "14792", "9066", "1503", "25039" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-11868", "5868", "1573", "22308", "27942", "13220", "25806" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "6498297", "316060", "-6879700", "6560750", "-578892", "-4785342", "7293532" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-355167", "6077145", "456351", "-6459738", "2475530", "-2801348", "2836470" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-763391790", "-1421479470", "1842219939", "-1396384047", "1279169338", "-974221132", "786409383" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-701822471", "-624495191", "1950360306", "-937735710", "891136351", "1299679601", "-1631128813" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "375062411", "101064405", "1860934989", "852100817", "1489729732", "-1386032321", "1372454919" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "211103860560", "165685254110", "475179443594", "83982532429", "-110181124820", "260080395391", "396030998948" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-345799561802", "385403135660", "394672914473", "-41576529948", "-89134685382", "-3611010438", "-230138775672" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-44213782783569", "-20824315185336", "119392219688726", "-48280571708474", "93590107082594", "-73857195319550", "-11092149545600" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "-3391364501302", "-76804068374074", "-5330058597257", "-19337292494715", "138572773907931", "21930262754355", "19745392677247" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "85902098584199", "111873133532349", "-68545677680071", "34394580702899", "-129707208239806", "124130433864871", "34089590991835" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0 ], "DATA": [ "11547754029851805", "13929638569741680", "-35234779220736599", "1556424996390195", "-9941069445592524", "-10351643092326233", "-25407298114074662" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-25715001828139556", "219757435776379", "-15386649204339974", "14219377191186674", "7764182966113153", "-17037164141869666", "-2475550776888956" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-9013210997202018091", "-1797035859465177385", "-5039026628271053798", "-2279961464871578084", "-3332468041413232867", "5196784857810113987", "-816043157177424446" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "6230937120482135875", "4406903786753583285", "-5727776665901788833", "6110238893930148669", "-145374459534268305", "-5408134830337680351", "4578682454615398942" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "2280996778076363880015", "-1105914554647913546919", "-2340456352988668320804", "-2286379749373826695506", "886840160760479118799", "-41094253890685508209", "548649222178985137197" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "2101991590910127128698", "-831048801347409813644", "283006935997353787913", "1014149330696637995585", "806441932790678928581", "1292353558196559947530", "1691441804823003410715" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "514147725545916528450", "-193880206083679947659", "937498535568154555134", "-112013115494854394797", "1838267628545853841229", "1697251710819151266567", "1334759843181386854916" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0 ], "DATA": [ "140211044419225416059523", "22798773775711118695439", "185167441622179764743229", "-341069468014651486949028", "-311767107447079981815378", "-365574085382023414765745", "16721445196171820250327" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "486153544869280373342335", "-242241326842640368113620", "-562468626770589338549903", "353095362900612835555266", "497938272554417006820815", "90302429955937293267553", "526449366929722769596410" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "46664866598514825254083995", "14396224856981525909820752", "-107662311974370582851318011", "-116315009943902083038882452", "56805605268567131709840635", "-285698053798149060391610", "-33274177545126848030882515" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "33427646005521871613027485", "2068649850038034587145992", "100657143587737579826250182", "-101842456842860935986029777", "-24678241147837041282536443", "-137081933839393292380849579", "80701010466319784595704954" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-125204491775364926553434405", "-68666196727647600858615388", "112354660219741185829905397", "-7303433862000761726317213", "-11446693224295095219494490", "-2156245347683069299464550", "154672444469077398685327566" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "23329815819835763885234375029", "17840398455337288079751030137", "-25733526444924201365089723197", "11519925701526868606303914066", "-8122188835588210950596509581", "20364966273892581445915895222", "33135945479485532338472222967" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "34850681321476402094647761148", "-8083975349781433603028632490", "-26785483364705248388507635494", "9143502111139064417571274857", "14109565008946975910759730825", "23146095454100851136277162012", "1964738793442941286156606155" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-830794414147442758629680858917", "3974824330694591996054219287782", "5212798423257109428476944724672", "4136148464931397492769112444980", "8716912509663310300081342235480", "6130432387446754799113484089575", "6936235167270048720910192819573" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-3535004387728242781406510654821", "-2981058632797045086695893841072", "-7090056840341574165394710895073", "2937436279617800594018353925694", "-6164350919245055090956637473538", "-6513208139095775762754457986006", "9324466091637799474852790812941" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "2028288650797899768325565085648", "2900055817977855811722193499494", "6905622275677150166335492710080", "-3160034744597195167509247764978", "-1770329430972443791351959666851", "-6751176363700567085044496183977", "391291090038019520158630200495" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-1144504652616585466559731200361813", "-1411714359592296247517000426578522", "1466595389854363706316815721880003", "742237093197555043514198686208810", "632837196426513796812458196446553", "1024964501312689171613773158523485", "-847167208284549920541560580297353" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-1379322307328006592168092307072132", "387617426942146023411122884259110", "343500850419230433556982280364993", "2429014391360178084347952076151804", "-1271102713054563598822742297111938", "1935501224421101911088894464720498", "891196511387998706976306482945567" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "327017567565957860896642013563192207", "-197957402373048855115227470112922554", "-274157035197732503264012623953912376", "-403813151150633640598242397849702978", "-607653760517680240576727061740741951", "-484513780480873333770895576601263141", "-364508171932424706077901220225164024" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-655778660285369872582403556455076537", "-305967296614644034918759561910261868", "408652192029803274954960477706952597", "491158410408462445362863360612553421", "190090738690396233311384708778934496", "-650782116647010450130445692263218670", "466999684855637663153637496446512539" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-120922442637350268547173038619758155686", "-14798309877377243156606971473100481720", "-158319052033044572444106251098798964128", "-87790259877385234658887405184771282386", "93922935281981262808730125220213226026", "152362390413913740334911190667453959896", "-144716399672639547866469402888393372244" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-44740461288862326679317055888851447788", "-19261460967897542219529042766754216418", "32572085694274520626483716255202096142", "61807114741875177639257174914455169635", "142136086887672692821719902247408741632", "-53665021237760509021660371077876689014", "135482047221090055561713975617737312735" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "151443922013736797164062148265526547401", "35894129120522936623313378082863551111", "165517845554666361071462006124502354697", "-161960863802263743123033369079756629461", "-13661079351804691514638526655907412444", "-88694839241604621649741179075868872513", "58239606879205464966310459406863836884" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "21865", "-9513", "-7941", "-4293", "-1219", "-10549", "-2196", "21139", "21478", "-6001" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "8460", "-5982", "-23734", "-11998", "17806", "-21183", "13348", "-23465", "-7244", "-15957" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-1268458", "-118370", "-1893479", "6108358", "1102635", "701748", "-4079505", "3897123", "2836441", "-5694828" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "3354881", "3381813", "6090423", "7476959", "-6152439", "-3812109", "-4999299", "708401", "1857502", "1616326" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "1829093763", "-4407926", "2068340761", "234009195", "98079137", "759088995", "1573936081", "-1886084340", "1660264264", "-703900633" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "1724311878", "-1770100972", "2005560287", "1002317505", "564934293", "-1056620575", "-1507809740", "1202563854", "-1605647773", "965485898" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "1993753383", "-1867185966", "-1797347612", "1892035404", "-1434558470", "200775493", "-1667955972", "-1185419659", "-1021155420", "-1017852104" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-103795691472", "50179936568", "-305052039589", "-310642809394", "298713016856", "430578254618", "544157970272", "161717434423", "-82568595668", "-69760173661" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-375428789801", "229743413651", "-385129101606", "-341027447447", "-292292749496", "262095081246", "399108175194", "-436877011553", "-184609205075", "-525138717918" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "109490881395508", "-2794346701792", "-136768254893717", "98238674344928", "-71024502877527", "36337204077446", "25777777072393", "-76891055436936", "-20658339677220", "107548266692467" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "65741531641303", "-120604147256920", "-71319706911283", "12568908870250", "20874672241429", "-103855183735668", "-53956000337623", "67466545861242", "-40824456528955", "-50502795434656" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-30593444826663", "-105265259920813", "90066025218056", "72114368716596", "88265148520874", "-26982520274013", "-56277959562553", "-46821369439330", "-84272522150036", "-67142476625898" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "22942929546037422", "20172954267975860", "9006547764775266", "5211508666269417", "30624269047382012", "-12672445409868765", "-32303326488011005", "21742501359293496", "18870493623228669", "-15114725655588776" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "7991033936095510", "-25436300164394417", "-4116325708088739", "-8818216017431285", "-18277587083906160", "14144228659002687", "-4827602588444601", "-6917969417697944", "34390202143918890", "26364020049473109" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-5565127045385194966", "-7684916584653270186", "-6526101704890152726", "5758616101220042812", "4277401219812095069", "-1156200668055380488", "-3665026910524676836", "7227753205619980282", "-9117970762813305675", "3814443356076271226" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "-4056124162849183791", "-3091054243487850690", "-1005186283000031535", "4264379098205926012", "-5816101399981242529", "657803495947876179", "-7620099954905920196", "1165400408553884079", "-4242137285679032126", "4883600219746027602" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "-2075689333753747735685", "2356020644862916379319", "2167344894833235655242", "-1233676921087779374936", "-472138285531110689106", "-1007733287828797731354", "-1150820475775440441805", "1849909446162084231324", "40485887422211501292", "-1531296877121202039185" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "2248262662434579442174", "-24067525590648914392", "-46650276984820791770", "-2184543362428055235823", "1974794946687090550573", "2197581555630725738513", "-1036621057391423397578", "1122920389059697318393", "-947702755738865994434", "1634754840733349282174" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "-859921720417617681444", "-2190420634375059510979", "-1434691179630443364753", "-518741106894841329345", "1093338259353051655764", "2211352717790797000825", "907793405354409516774", "220145636554287824727", "509691887514400022545", "-1294594023424323107387" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-526990487436843194807840", "596575213433002438735038", "-71236910029025414107990", "99178925759657972243233", "-440284224066490812643561", "-428326164367815704740490", "-429553654939143998439803", "-301485703325840334321221", "-205335077080231282419987", "-586491652024392408421712" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-543688845315372053238708", "-163909849043312495770038", "-389689726424929488513817", "-114895253663121558845496", "348640253272437667385550", "16575128907035554845887", "597210944202677679801832", "220285812487408184784084", "-42608008827610055995030", "-96942244128145461390288" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "143527171537167288568334963", "63922968735872942354971347", "16148467038136997458380707", "-153693517794242110028457288", "19318376645712741297662190", "-12794255459929906566054319", "-37650752907879433922942673", "-150166615652066337683957386", "57329802738501405002838890", "84897963838366989876933017" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "96124972909010047517011683", "140953422210565403195650941", "5311198421189536461110037", "121439717040191911388251790", "41648789220821764736893059", "-34464297128753801791226983", "-130145493103864535084592015", "85478399528142555833134891", "14109865387017479696802124", "-35883514830521563515868006" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "7929873152858210171340528", "146946594934440045255005503", "-63984879531222004851746641", "-98404545568865647913143114", "147368438153309689359575404", "-125299622361117017666281396", "50888897645607457371420920", "35436333051448932231106760", "127474433365288821803863819", "75480346343825103775154749" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-8746748457647904867496756699", "13886890262009261987702677945", "30610269704200657332639932690", "1988011012396012136828019533", "27809766702732680733200157750", "-13397700579544534835060124865", "25003563641560620051585929625", "-36124205442930485039742412898", "-26618151134121364917297218959", "37250266093672499484998022951" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-14142713495768494830041644219", "-28826123233012884759325086616", "2139184291361898827778357325", "-20691297473114691371314913993", "34039896653596475818834199231", "-1467759043071863209472101520", "18433718648399999914637264302", "-24512577345718204022326385307", "-17460565004252223611174273742", "30220497605213358105564998501" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "10121526385221565224067878565679", "2761189092075112373016075500931", "8854254270554901165807339966360", "-4844293746135774082587580385559", "3690381197256143789522141159908", "1243411595693866754538784391256", "9189681722403043058688029968097", "-9423820843284925151006522300117", "6829156521436853695351870360315", "8184630808925606776348432717524" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-1198084970702764472492358102739", "3152710836296673619366574627547", "-3203266598150150519017435963345", "-3401665769768522116199310644875", "6607270485165065820523048271021", "-5180376940463540157931412688597", "3550769633626957920535415329535", "-3923968524034696093714727315085", "-4025531993361587655896041896848", "-188450492479152136768542630378" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-9632952156052895819744271379361", "-4884426114142676161683078815655", "1839921941370111674080951268455", "-8242503028303467805329769377593", "-8839943769048065292932997859326", "-7067251768733441732414994112967", "4523867687328920768404651884895", "4871179227553544008972518619850", "-6057694139116711199472594493097", "4059555045350317377455403338609" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-473059709957241261159046271732783", "1980262112011223122009659766419299", "-12462833368369094491640116529143", "-1189936951170501491703143185809120", "1637235101820702334443602525486638", "-2031029039226482674092805839096008", "1673643328081327255616083846013952", "1515455878435179342948918055102912", "-249177468182679514366885827523443", "-1462399863102676315035000336009503" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "404343261308094133306771010587258", "-2509027130906219806348103780125817", "-2235370619058054613214526829812775", "-2180139572932375906797575417923006", "1786990964606634697851218573276596", "-222676010348749137040908415328994", "2148962315756674060907600200655578", "1131231890777254296826085678598694", "-209000893739050027703488315683250", "-2491331756189203304481882066855639" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-185542581219629399352774704587796685", "-394902121829278205437359525805807857", "590713072897259853019691937013503396", "502758727281988975427078318543416657", "-443747842243734666697335489578837039", "5042221370262609425633645769887675", "-591977482927777323690326997409238587", "292422542569899682160719976080427761", "605833396059257331402310165203375182", "-9563525518418491930653260896379776" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "334166123380339702684858383425230913", "-637729563446579910264265173398457924", "472769213866685799294818860568678399", "447146791822178801527521581709580511", "96610773493845656033107109605172170", "-142602384805506729332865726614124340", "-310601730706741597007959967153005294", "354280041691278605239105907920727029", "320355952808759932183649996729527966", "331701763416087509468808713667332601" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "49073461277677362785760660857736919947", "-169416908345824675858705165355626189812", "84167001824144890089789137366638401928", "-23409106148842837235476212707694687403", "-30271474341226497841818553413331361753", "-106504152229661458377059178821397168599", "47171294945627565796345213643071625567", "22970191271966763243715811330493877198", "14863926779301434690221505350909619916", "110474698934476487574067856333615082472" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "125849800007107590516369492180395552578", "143148433387379660832193328066338030792", "-72853653543700073732079719513168773515", "123778760286761947686806074556748610584", "-105565654733279392109260625797964696434", "83264246581235221617996610103577182490", "-48415572079837050298521191611988793535", "55401352808652663867123392021289274501", "18767844067668706651480236415919760235", "149963850691371825124263493422131099303" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "125053867195111854928850289325494913185", "-141558821655686526589894984641859254355", "-70235167406408960299018082636911869127", "-93931785069493972666863586219173004358", "-33652465762715830748074030542843712877", "43131529119131579740991924094501938303", "-56826490505262593726163351863722744454", "26080328815831670206668453834325133999", "-118301668341141860875674987759645097183", "-137560743311784101467304719508764176954" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-13389", "8685", "-12385", "8428", "-23847", "2877", "27301" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-20601", "22726", "-19831", "11763", "-21024", "-26976", "-3495" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-7613329", "-6191296", "-6562342", "-2775043", "-4626420", "2253566", "-1356456" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-1881110", "-3472963", "-1986841", "8110500", "6613282", "5706723", "4659122" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-845165868", "-123936040", "-118407476", "-872134111", "250077717", "48927889", "-1242740962" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "1257436866", "1664895293", "-1592574645", "1054532878", "1586386182", "-1423067232", "-433225350" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-1448117074", "991933747", "-2007741814", "1096958894", "909454330", "1945230524", "-1135940304" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "454731687426", "-410445542500", "-275815453925", "232188471199", "74176210838", "172694874896", "-432229161881" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-237742907009", "291725362921", "-375456133851", "-172555537749", "-77222985727", "75467983132", "355603655065" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-103229358570432", "-52384313969783", "125123784330760", "49147811785670", "-76188420587913", "62077347858917", "113236111966942" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1 ], "DATA": [ "16839675737350", "117528188223946", "-83215066360284", "-106240126895133", "-140610775675863", "69161046340219", "-5367620164209" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-28041939750005", "-9849374173718", "-104280463802980", "68340765792259", "50866601864200", "-124901901785729", "94320105961599" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "12964636865091758", "-30900961707076249", "-34235023432781587", "31452849974220059", "-32116433493349645", "15056454641844697", "3220380516817924" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "11565990132652309", "-1979115219549902", "-9506105015296441", "-29947064953359026", "-35723741290819777", "1692216018015076", "16174544022195084" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-7310963412698136424", "-165597943071933948", "-7957288198886101326", "-1416681132847790179", "2279955654231160293", "-9208861139945806119", "-2871834669649394603" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-4691141329322093701", "8948849172004914148", "4989049806853973423", "-5955920418702428305", "-5005462481578120888", "-4824298625293485550", "-992020992384580885" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-27391602612782822280", "2319008273932661576621", "1379487111365773121977", "694284593718405260502", "1761355855168617262841", "-1515770924964548454668", "-989979187498153392043" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "198638501199435366825", "1125019815452792395133", "-1049896359176778070473", "-1601152615237961695468", "1050439216436645606582", "1663568252582913300370", "-2061203660241370183042" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "2113036292339854435848", "-1939642742757517734568", "-28192641056500022357", "-739364128466490143441", "-814580364884709918305", "-1154955975185230844627", "-1921376171526945915117" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "155234916517253223214669", "54569816973154769361790", "-396264903716832274146123", "-233058506077293724737866", "-259977111446647844862201", "168457257727401353456526", "-440383324891821228732709" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-317523940996880385737765", "-93963745003535018073676", "424089378343927508686049", "-547783720395248736053345", "-573590139412320114240415", "-279824181416708109089356", "151946399597755342049492" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "28344600382890942689933714", "61364243140907583311935231", "-154265345511422448987361885", "119378359869607162352295161", "94263708499987060050544722", "-133072175861722803758097907", "-11704454779169713276909351" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-98751232093166611752967870", "-139312664474034269271128429", "35063444117900395179797093", "102516806810490076849188468", "74192717816812033285854691", "-146743149862829000033130770", "-141832950190420181845392446" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "78201965083863028268161981", "-9069855773037171038336750", "132188050847166348693715478", "45312541275344755216581315", "-57072509286905908805327196", "-17639505153037819959882217", "78921165012102074114569802" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "1613854197834032136730063430", "-8470408631766558968143580401", "-26796998771160976867597896819", "-13968703974822115617423495877", "16391297684868529484428246628", "31996253723693099758880092348", "-28515449027235056800026707822" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "29003901243034261051999340651", "-9217872751721264488488548966", "-36037333564320399226575507178", "13184620706272928711369564930", "-3484312485760312610634417682", "26298119669598487256561916979", "-10337073506268829385902024212" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-8395953463471428775558712544937", "3280829292043794132363911327400", "-713339165879923703523124951611", "6966592500081948880026911531684", "9458860222968810665112995514695", "-478657998044149303771109777456", "-7822981040467488960380110589645" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-7687380421442439073984466158397", "1255125729759094668996545862678", "4087370506300493221899403471926", "2239990096185047591766535855780", "324855577134097258958507078312", "-3747605668465760865974870493838", "-7517467382536093238063723586773" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "2264136733622961048042169712287", "-3680763311587562301144245715244", "-6085544815496196698212373259054", "-4725158409155890778538182765011", "4471247291507677347209093541425", "-7596184803597155735867437286639", "-5336764833430717836720119834307" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "2366929765190575517610613077354934", "-933660721467627547628108668944304", "275402050086771070341080860167297", "-364049334222814377243385829475539", "1802721796043278974720830665216534", "1876897254307941830406818341784439", "-407308419988080444605787064480067" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "1653560379001616793860692123874411", "150784992794639309979528542957154", "1110971968475734226805305846564008", "2480908389064352508977069518804243", "-271941929813911201105163823221822", "2522624759865770340700105525537365", "-1998298580902951627476767136760592" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-303303768433761525871131397626734735", "580204425169654418247447034004051762", "501559035276733743240608210997496006", "434014792885862484071273298300463825", "103329211286006486949233240613271101", "-505666350544432180911151793361598853", "64165233750039998288773750264121396" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "424864295799616426484212056964871888", "-152810042904774611567602887301612969", "477355529458883774205927095018255750", "-613775486154323397471667167311948258", "-30862514129841545507008489633366896", "262491469819294261426016375408380504", "128848914605896127620809659768891190" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "1176728107084355805100360783355792854", "-7236188428562180415879913333530957262", "47292783010039120381519447577145042690", "-81565099348678451031823780150331852600", "78688059613057943024694270282505199422", "147423850372516204461538795771199943752", "-82721829556506668110250246860473354784" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "151022194687816907306712480755439984350", "147565573622060292887781241703078453904", "-166782516227114338666555252025171372201", "6746685360027376523977385526199820516", "58540530404008066096527055300466378045", "25319977592399722647228094404431005983", "75030966136806031996061647057535591049" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "121830200727005461151499890045616240332", "131014485689583600627833063453380752758", "-30720940443795603452473820184705126436", "-15668097962838507353292677213849146239", "104461885360038312673700885512336746198", "-119938884501349847589471389439448814622", "55172679667339024457950426429614765191" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 0, 1, 1 ], "DATA": [ "23120", "-32572", "-22215", "16976", "6717", "17948", "-11665", "-24766", "3353", "23320" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-1104", "-30742", "17502", "24977", "31980", "-21331", "11269", "19058", "11761", "-29057" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "8257019", "-1751304", "-7837619", "3010154", "-2333966", "-204198", "-7074295", "-6464090", "7365046", "4895822" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "1458664", "5699996", "3620616", "-6624045", "2550814", "-3036255", "1587012", "-3365811", "790235", "3027254" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "442486938", "441457258", "-74436116", "-923568700", "1944924352", "-1941482279", "-1125035682", "-102135544", "-1196111163", "1765877471" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "1765031209", "-1507660239", "-1844477793", "-672110186", "1975877216", "-1203240404", "274438170", "-1671550411", "-554361568", "-163722651" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0, 0, 0, 0 ], "DATA": [ "-1263789103", "489447307", "-566090243", "284044302", "-1836581764", "1548735264", "-191701487", "-173609011", "-166293908", "-1503453842" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-7765917028", "-67711994811", "457867171231", "335464190692", "-319749552548", "-252965983097", "537149590956", "-106161385680", "-489362544078", "-324766745567" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-229499927632", "65253348718", "104144370442", "-291236114406", "397153316552", "-2446986870", "-123088300656", "-203970057575", "-542590736363", "-190268824826" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "53278510251890", "27395352245810", "43165128890490", "-125549371097904", "45913496601859", "-88777298033268", "-44578278857847", "-63182483121634", "-615189926077", "-76357287782650" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "125574339986876", "14897658308787", "72773174883515", "42123525241235", "-92023130876220", "-137779513626577", "128721948417075", "-131284549575073", "-72682195461575", "-92062818915183" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-56667925886322", "48625292283487", "118021699876201", "-129645086891796", "105886580754602", "-81412745473268", "22079958372753", "130176891204108", "96533194170193", "11884025784665" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-506532673811626", "-35650957401620111", "3797359512863061", "12274676970376749", "-7643350403625116", "-22952047899787933", "-31801747497539952", "34825544608002068", "-2163219849477806", "-26717933056837916" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-18117789352537263", "-9588508057343771", "14147544466407231", "5358112868530619", "15194998186633362", "-26368440181218293", "-12926433949412576", "-23715017719955956", "2124625137004533", "-28761382221979366" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-611587277931329314", "4788391799910085501", "2015606813848422210", "-8162441603711661172", "-3434406396031032576", "4201771398235074913", "-7185844575809467367", "-6124286884318139459", "9104770916622956359", "776894361998019584" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "2442968139182218751", "-6191133470861120460", "39377695956604263", "-1911371627459726404", "-2032902584307265533", "-3918625665352066431", "-3011351064796167752", "-4761219044304366605", "-5873710765610430386", "-4865925097841000964" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "254629702353421818300", "-2201401895035914287579", "162176927797961571359", "-887977538520723081192", "1559680077891008200419", "-1522754607817991562870", "-1961188309718634913132", "1306926884755953118990", "-2195000881232701833676", "269742032865994089514" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "506826741901505999759", "-2184273395087508003256", "924141038875577992170", "-206907928631225444602", "1962143049953611087716", "743436553398815951437", "931022352993641629959", "-1854695194903132671547", "-1981929463828562067483", "-276972122424764490843" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "1493690505338340259685", "1586809240658822057888", "494350090577058363533", "-441261947278991503330", "227960094120329115559", "1848792322278602742922", "1557617272354574911225", "1731256272922066434300", "2146230871407347076666", "-918839131781624132904" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "22548387681410857803984", "-486740942160285075283728", "-18757726112678919772843", "290325417808337495708055", "-493613286382529175399469", "-543640826835836745923553", "305872812141626487961328", "182680085006567454977369", "-381084632278529532946167", "-438390880171661736974263" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 0, 0, 0 ], "DATA": [ "456312474649470033816393", "-115833852286009708732615", "68058901001059387892663", "449536434823641408121541", "550535224734224663866259", "600242006088744661574795", "-474373221742191222036625", "-240661321200669878886195", "510976975891324563556258", "56581084876567011909938" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-55030941453838611172063173", "154562453921435135381589722", "43654862091121183246335073", "-37043561098272538846899535", "-84012525778134679892504063", "90521181578289106753547859", "-121101532794400375638986820", "-48962126032349923649363889", "-49416297762321316497159289", "91097613555149997187554026" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0, 0, 0, 0 ], "DATA": [ "740754556792408749652437", "-48364577771068363588552848", "22689046979811309216159063", "-49805825990806434744728742", "-129575325280951011614339770", "33686330327503040241226442", "132098550234954179034221650", "-19568317425318962814852019", "-85488236163679977350384849", "-32900134218496126753349349" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "129277452246091321049329712", "70220503267517710030799158", "27372362666191002140346309", "150017068938796641247123814", "-83181229560758393426889380", "78268145282577350329747809", "-92583446733322174938991138", "83334736171733181604682678", "5792916388018467792831905", "-79110296716121790367484224" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-6762857221159250494826450291", "-10332119938961796101160722490", "-6827819665107707846347452358", "6329384513346831358274334163", "-35228914666708609766335933234", "-18048951629266414989346849863", "-27226935845799367007517234093", "-33993966153233641238453751768", "-5449254845364088654662599852", "7541505715737170338078113293" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-27624667007791025693852828942", "29791087942990315789537070065", "-31210197914933328681784492817", "-20492515931057444475484015790", "-34897600909921210666914133162", "21001866866717994908958026000", "34533628665541062590101591605", "9988772685214552755442833227", "22086548704119049555142576363", "-16156072487413105847081955974" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "2364616944071636050694451730445", "-4605392974112860339491707249622", "7762016214564113231716204977147", "-5804616094053219373113235353220", "-7149030242509106300895187420705", "-8733720516856953446393989573010", "3766997683445225481997098610374", "-4681729003393749052094837537164", "-3858591495312700256431295473053", "-4277910894337725265278367498545" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-8345846769042618988327927260668", "3034158729600895891904384787679", "-121178995572767479680968825369", "2753730356624797067210787714567", "8954963579041823082665982027392", "1270742333952628369439000703669", "-3722978202569846960374550742749", "8821702309083726610115002889572", "-4646910584068123191099120325769", "-8235420195270182384004027445084" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-4599292704126802281290827035912", "-314623101152193205334021409871", "-629541047462259285195633937282", "3573126210904165261265049508014", "-1688995622757664479276042953337", "-954432643427450256096079589970", "4799594864439399541828732234249", "7068551937371368971824412562795", "2938601954614403464560201414377", "2583981951692907177619688538873" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "1801390123686604711395138083864347", "2408175799973869350806431040308722", "-1778542198055167111435408713744051", "2503317898880974086644685985961064", "-1868573144190196894242583180400118", "2413853351573616737379149363424791", "1791133303754307763813804844006693", "1880584363502969839626017340127067", "2319612620308107999768959466515839", "-1312849382673356132056970324156720" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-2310170941545151210165819902499827", "-737480739959391674903038969492449", "-2309694820066177770799513479215373", "1423472051904245134846442992358971", "260457318140005904743337186826997", "-1594690225193818094407079633825817", "-2186698024851903659624127278863647", "1502160181086743651783854079119431", "-249207087322062171035117292968436", "131669942196137162081550281921265" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-221840655914556742984533980869327276", "541288680814766363243902199787661875", "259521991726914299693855180749498836", "-642098028427571595255754897557274431", "102852117125804159452269997724256795", "-619484029498534054293488403236201260", "-345175355591714711803667121166327599", "-301314861888172625598139699349559259", "-609730883416842409062992565799074655", "230365679128699815563056808949145224" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "148237310969220211917376683153947791", "-210879956694236410559514093773799189", "68825095267034031802766092342803122", "21069100059826162651376609239596285", "595526602428177494983219729529252005", "95671763294916433447636192505612543", "-121882858245568611085988770451861605", "206325916742129711002135164503598699", "57651974744686292971156031485809986", "-320734725725255645082933245576075729" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "-138584416420268103904526275311550821586", "5083386037235063597175109391737322324", "26128220661314654796494761140594032479", "12192498025672996200511955739264829627", "85947824515275949989859778737001518967", "132920098023348124785802079171904221898", "111246451182495893420608179633240106537", "-4547602311363102763260302847649859534", "-43987674707274853293509031074465543646", "-7024578484047164009719166261798276072" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-98438730789672041852593692005666781163", "-50079767906520589446732692469907844466", "-161434501044028603998001795580533038376", "137619979466523372085843169543259266571", "30806070627846697101821976362585151873", "-20974889368336688743196860981435677443", "-118421326318547135923082579581184125588", "-36658255136000073539449905192106631248", "41010547609721177330707111502953515617", "116618718482647675602042707539708842430" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "56783177633153952864226073173557138209", "-106939684114816004877527397111208670733", "-53010195457974371105397833315577691496", "-71944649577959370180890395897943740754", "97436272237031860506529444641492486685", "75452790723745404169344305366343804695", "112051243374889808475698084777418001497", "28127293814479165375232961041795735138", "-79340368891394516117475712925796793216", "161366149054331422136847320127220486305" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ "30263", "-23222", "-24262", "30483", "-5838", "-11101", "-7596" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-28613", "17283", "12286", "-15277", "-25348", "5314", "14687" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "1641157", "7771466", "-5863310", "-7519258", "892965", "-936146", "-8184473" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "1892203", "-6603897", "8118473", "4954349", "5950071", "-4001964", "-563295" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "2105362262", "-569111656", "1919017986", "1876055546", "-1146939193", "620406159", "716539407" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "2083506192", "-1628538025", "-479438046", "-1422963534", "736679219", "-1232279592", "-724157284" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "1286129607", "745445705", "1008263313", "50113482", "-1025981626", "-96567663", "-1070688111" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "418265202101", "-131132103981", "476092459444", "-88185972192", "208316876344", "287012809896", "-90799129473" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-364027495907", "-518301781733", "-87293329789", "131975972315", "532952706734", "65088425600", "-389552284897" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 1 ], "DATA": [ "31618280405118", "-136363944920194", "-24219400967405", "41351280642367", "-133525694845951", "33222176685107", "-48907606129826" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-74180834940463", "-120438687871068", "95491516668458", "111229832917865", "33731924194747", "-126511275125177", "-139437230946699" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-81052755845134", "-75947916229442", "105789356551279", "-77360764385979", "125756275622911", "92621084693706", "-8993873340097" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-218063399041728", "15987427800048505", "5452953879269108", "-2907032193248868", "762605768430446", "-1095257289404828", "12083366342031533" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "10657562176134064", "13537179043764834", "4147191201684311", "-15913012985064112", "18305247661337724", "25826108878037669", "27185142855300649" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "7185681950550884166", "1151981239380928488", "8244169834678820702", "-304666117465982921", "981971288330668381", "5795574764637974961", "-5602631342767621026" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "8858282230609918710", "-5845597164953171622", "3853395376773770978", "-5420964350807318652", "-556878098560662410", "5157835722577117011", "-3095549053985906766" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-980234476785800984103", "1785410146071585960280", "1741125755543744357745", "-2307966711214941804481", "-373078769867949398924", "-1086641478410581865639", "690547933962974045594" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-885097049671576555174", "-1893945141563298628450", "-1652678503714833255734", "-1522111312512895545246", "-1668888046878977777801", "1230021831666666579759", "1588026688627154194388" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-917162439303146482401", "1294154478309086604917", "-1353063508538165826259", "1022085074719266892796", "-1562172028448696446044", "1827858998664290349067", "-1884237502299830467832" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "-484334248622047005148813", "148191614369576787178350", "-41916037311516330068416", "564042302843825437811083", "-223411708998089161582419", "-493169217694889240887888", "168822761929986834961244" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-76559387748666164545163", "-9254395158022788547985", "-533412485676057805450960", "149982043973998367780013", "331053556044991173393445", "514169254104754203169529", "-24148845097144387020494" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-110794762039637748701547811", "-112781226990835664252750191", "-126691789751592065497849314", "144891347683530500307750752", "-95002620477467921055244746", "40308052656613620648154342", "134142133925977560550824382" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "46547963198244968911697774", "97558079483281440143910654", "42302799957358224994647507", "-23653725784983135812047124", "-73629112624555133955638884", "147336720269797653946599523", "49289602849638790286981280" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-70628982960116517292670785", "89897841125429743744022708", "40246726976710463297014640", "-18507266936458323746034023", "-127317537089789264088083641", "147526034684993650522315979", "104950772823463920929967725" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-7959063299212980364945888769", "-12214855654426558559365015380", "25951542396314432482289253877", "-1020221313976375715276389621", "-23858719187545844530085394517", "26756899139375533317181309141", "4149356920572734945386012056" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "30516050835182806794120674336", "-24853046938355580136642360705", "17673256930214912186178705872", "3794080846648515873847989399", "-27519999940588367143793580557", "-24055845379132928502612582185", "1667643406878176303891605843" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "7481053395824836436290423884846", "-593931320626780956465988943779", "-6000659640663712751730791207087", "-90100969836256549622480172858", "8857339616569823715364996577725", "-4550234737408799149611645636613", "-5298498692956294201665781009355" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "2183123309024430871493750993081", "-7601532421099550450707534366673", "-7300980343536532697821828736934", "1354900935970415000060545678696", "8198340038238087506252124691672", "-865479512261563494978410780512", "-5046161384947842589369454777111" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "7048037667368946471394838100018", "-6071852502042552382040623004150", "-29904067906303750173605027886", "671351089519547849465678121449", "3143301561492705185485961790751", "-6145271153767518350310952751439", "2739965492773135139400075441125" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-2567371035309291572933015418896592", "-1613418673846353226340454921565057", "-2365064503966708848862797241011316", "-1542776028214362902586142929151475", "-2077125689533596049348241259664237", "-2395709824654377963034101663677500", "2440524930359057180265130201332264" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "126271990461855730642053909769616", "-2141603070621807963066899994069276", "-2176132802609488796741574791492255", "2332352886535253698579202178516320", "-1988660127485179422598316866522748", "617466151275332301720527118576755", "2496619525601425389231233102949834" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "231635227467662539191613532700092322", "357886376162746208586158220391320643", "372166497885582630320382738692891137", "-387074150560768584146102878863840104", "-397494095284070251215509382375703502", "540335898477711720687165346731799594", "154690828219053775405144687447110697" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "656778981191586847022137324312215960", "378543348336505847394761525402250912", "-170070094972872823236801280112198457", "-10242741317397583042355113808738618", "89873395803487425288992885513379780", "304757291680949813380778307009872780", "-139647342743838007971248857009724115" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-131203220135616441025612961636698464749", "-136533237651213770170219781290216699170", "-115015389154631152533856730671459648641", "-94487920510079112453883494033826826524", "56128697101429071079534166874945886309", "62210939351804546691896889001498845763", "116499323952286447489475520935718675981" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "111388442425565536328400022570545560941", "-128274800782884675622442061938835593669", "-21259026684892744639073607446054060342", "-112426553286465994874443695017393042924", "-6478832771211729175837331925924806054", "88007792352876262937416934703934442548", "-124891073986219943979478446893457381063" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-53247472166305185804034744604113912186", "63873846134112392155257205697721451801", "-44185307140014507458077369663628065550", "82459261534617980418389187926742556129", "-91555320299720623336768030419645817723", "-11478808234125256915529445366937625010", "-284101804019488359263241192605820668" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-21109", "-3732", "-7946", "-2257", "-3925", "-30469", "19503", "8362", "12991", "7380" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "7887", "17931", "-27968", "-3196", "-31776", "31067", "32277", "-8764", "-31456", "-29467" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "-8341014", "-1946977", "6040551", "-2571346", "-6392818", "2115232", "-2447337", "4046082", "-2062834", "2887198" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-184184", "1975807", "-2304689", "-5917706", "-7687940", "1428040", "1997257", "-4160391", "942644", "-5375074" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "461428707", "-1396927537", "-778825702", "-1909514860", "-890594087", "-360960780", "1809000906", "635847266", "310435728", "-904068800" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "1160190062", "-145728656", "1783609630", "-146939811", "-842989885", "-1768700091", "988218247", "148328980", "-2071975098", "670872405" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-824363066", "-368967389", "1685375538", "1129837916", "22659316", "-1334068320", "442107323", "-1555351812", "1240907608", "32207466" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-467348909083", "161337736582", "534040697288", "147508961254", "-55643966274", "338027786439", "-116692579226", "-269893444320", "196082834886", "251179928243" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-470003994676", "322677755949", "240704268230", "503601093424", "-298384610687", "400606205208", "-418170073845", "-133930730273", "134075361471", "481801156891" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "3339778666379", "-52347274197695", "69720482084171", "98924128655576", "12928263807684", "-6851555304470", "51780495955443", "83679464532035", "-127505675951073", "138274778683006" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "125929064682987", "113730203099244", "-100066179950776", "-85878295227479", "19299300138707", "-108539192136351", "99138900176365", "69954325267229", "-74544791461254", "18922888907199" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "45485569650205", "-128562887157102", "-97873632139797", "30278398120164", "50653763282967", "51088353337222", "70065775683504", "-138552521631772", "-77979604107570", "48798953674588" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "9248286127591027", "6920015266380308", "-31298699416650301", "-32620123808864302", "-8198521317945035", "-31784959087849283", "-13290835987057648", "31560002024245360", "15568532723510774", "-1887653675336702" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "6474236439552209", "-15895902060435063", "-28567392174760254", "28377048461337220", "-4788924744301655", "-35902541227577686", "28213667178040981", "27374686655345350", "-9347347706094784", "-22558945363830613" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "4315782565441521668", "8173082059740912464", "-4674157300377953730", "439018152229220757", "7259585234958472340", "3209551603044381642", "-2593722905832184497", "2027467253721370766", "4711777015859247337", "-1809157338227183996" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-694544658659246762", "-4156045606313107281", "2281142887622773123", "-624104445088573071", "-3987668248832946150", "-1357628067592529656", "-7104181881423591417", "2383273081562892502", "4853234144609282656", "6571580420114899404" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "1101678549357629021477", "2009977653178126846280", "2037345716315195820691", "-2096610127883926523376", "2194680562745348805943", "-2143848724610431766410", "700093503999313554640", "6460670107803288774", "1291134572995199955560", "-717095483870334127208" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-1449785013926244904068", "1586111973600917779568", "1975689855672951561335", "1946100167589360666679", "265628521821771152170", "-948749623912750799141", "-1084337593623251031397", "776286080436824555132", "-494554638041660841031", "-401685002986893137940" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-178940137819013596278", "-370142275253865396698", "-156303150104810810130", "-822078323048785455176", "535756974328325498532", "-890371668586461515474", "279474091115067270095", "2075769840894911144578", "2271112074461256742768", "1263298342536869880178" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "107390994480706624333240", "-272629086063015754481125", "-125381849980531312111076", "323365287095819927005598", "166918591791270045548611", "541466210731753290375920", "412426989884687859377573", "-373858895152711407562897", "349597921666518744534201", "-477064957800482399571210" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 0, 0, 1, 1 ], "DATA": [ "-350456949692319622870317", "105130553074956289619769", "-454855427531984650401861", "389828363885037529338500", "-45149503960310017108207", "-364809035488363237149074", "-212095481090572510124715", "208001365502923021156304", "-132773639805291311517839", "524948545150003066797425" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "35259018232949564354576762", "36343724900171873095814458", "-47507355485359376375717880", "-57533411745939815466380662", "-60092847738452003317331023", "3800735336914918445567351", "137812953107886858371793906", "34225766901520865002566632", "-132405725520797481081665087", "-153398838732410672130385737" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1, 1, 0, 0 ], "DATA": [ "128319155253991829531389637", "43165460069726070921482804", "85467445526390304512322840", "-146762181884606435611421185", "-126844884930818972344264748", "34361835173349993323133217", "-68547202891841302450583744", "-40167006696488837265069472", "-74901074229595686899584654", "-68749948940724462847250795" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-38290033550964318063135667", "-20967313541687634038075040", "-107264911137060066098734846", "89584407711339849997950109", "109335186095373309975629513", "-14494027827192960963596349", "-67509403216005821639193263", "109578027592798760144924982", "116834891439714631179635997", "109674768496669329805900472" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-13706828639541826056708910505", "-23779415475695806226285335701", "-20403054366535638844892685460", "17926682575342545857820109930", "-29969666635246903811911577159", "4322913582730437845509857109", "-5090037359725837740805001421", "18248885943719255743411625340", "-12657516075527000586763065411", "33671850711466671188705676643" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "21441226402320291786796151469", "-489973429642209279383578568", "-36568669593686005393185481762", "13796971801458472172520434088", "-7994017766016214184641016553", "1561490110301188737024730447", "38074453848227878221879512526", "-6611262569813139733823741726", "23022812151767616798249226140", "26695316798647697932393341179" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-6126422073163258556989145139770", "7257045667636323814171654348911", "-5274091940900155174972411849180", "-6812696260414552303993540333266", "-8500309156232696643300971495939", "5321769864659588392049817499314", "-5039830093949469211483714156812", "-6633027512060316645790696983007", "4340429228631480058324378799636", "-1630878746182595626723753451608" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-7314945488320426565820671586637", "-7406514689693791623767541892888", "-4830710931884289793431794199249", "-3758167604048365457370249362898", "5317862646659352563988454094829", "-748850916109857148485910907057", "-2993387002194928323024015231901", "4211954164724882453586630364110", "-7408921561114481661493286530975", "10033288108411665452743017188443" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "8053572570098281818356964596376", "1908605622493118035491961007084", "-5937699460286270647647349131356", "-1407164418808891695839274066972", "1026031202093754747593428440664", "9128075080231853627740826681361", "1150734293920781967788066491325", "3575593240656154263696537723347", "-9578952592909740118052990539663", "9788570458322925906731674034571" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "2546167711754510531653417471247239", "1864915514804198253374517101253374", "1572251731593167469136110804788032", "-1816600003045057303909283771795164", "1520652080462583011477916567108957", "-1734147169849373491334306258765538", "-2447614261820877423091669867951531", "-1826396544811346698392237371607395", "1502568816413254345265156473409654", "-2014610313148128366910629261789779" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-1855163614050249786123016316161336", "893899277550555639055767796543205", "686227439600851989589634986801690", "-1484196902391738874780681638450259", "1181153298481188649464625411550110", "-257877919422309608044093539127999", "2468479739575998772676829454099709", "-1539963498806165921795637993389302", "-2243382068201233838876165984414063", "491038991263905025700206542020861" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-597285041655777508836370117424546286", "-212915674112372713563243155334464735", "-169733400631355230665246923231564106", "-257695180843856835976546761814974502", "-159199708769938756688005712105595720", "155463228870223105562821637784150991", "-421604544463741431747680657332448481", "192822447775661213463129358744172317", "-286626526760237236497088410330567742", "-601517958441081087199398706025786382" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-82547444875501132761785352487359116", "-246593569791118450786584777558750625", "-91844724030484160072926648255143672", "-69252035825753540780599407037419460", "644781206633544756310104714559447256", "-262845596528177736487020213988699925", "-375729034479299520248775176155207770", "-479941467647966077592734215031877163", "286117330727391846690235616502481186", "-7294278388158531393325689448264285" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0, 0, 0, 0 ], "DATA": [ "-169130959752592548102532341751116281504", "82587899769437528472252758495774486404", "-77723031252037678358942277533026624307", "-39787175488615223575962629928173399959", "-103079418527774040874191095440451368115", "143404785392510227499220384378768451023", "98751717881147885089187606548239643742", "-80471675701993721480829704293444582397", "-13432971458738065305485925445592722750", "-68143198119846648770859620166719802" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "82277432281188444270047990740605736453", "-140504957823814927502089640850816061182", "48534660051854785030554289649407895158", "6422771152783983559395644304289119920", "162012152615078454305645544989583091731", "37269704291699978324341128764605257057", "-58248048235361087656770889990621980447", "134745944265183112641665968501631304489", "-9136434632101510662083878921980605716", "-11817844733230924939043201144789838478" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-17709917998817602362457338930665430374", "100705413630779255802749096858430372336", "-65652752408617833644830081929182653005", "78721980566008540939950320927524467430", "-157085052161045001481271504295196155720", "21700226768392088200691166712143650173", "89253038833049482190072700821475812886", "-42386752776532873392991410023518397456", "34328803512588938135564393642000374714", "126797458053317029746311576365125139325" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-3880", "28077", "4580", "16362", "-18975", "-13634", "29579" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "22466", "-19394", "4211", "-30616", "-20114", "24229", "-5149" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-896877", "-3561706", "459204", "-7986999", "-3891501", "591660", "-123828" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "7455787", "-2954938", "8061044", "-7704103", "-4283355", "-2688704", "-3286813" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-1733165912", "1441221142", "-711447900", "-1370755477", "-566470981", "-1689066002", "-1518943706" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1167592421", "-791885241", "-1679872731", "-41746999", "1369029801", "-2099305061", "-380693461" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "1876525998", "99057337", "-1107715390", "-1940677144", "-240324753", "55473962", "1797328299" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "295503234342", "231420286475", "-7245442623", "-506030630708", "112634015563", "18061760994", "-112548836018" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-365159103775", "12635282946", "-514650575463", "-389272826188", "-475262038569", "8128202421", "349784367667" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "105099742726339", "89319467966898", "-45683396076389", "-21990516959323", "58719877204527", "24673773662587", "109803579959540" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "117460253140304", "53310537831787", "-50260601769156", "-86329701674689", "4117449846060", "-101524170218096", "-39178939801212" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "116693494366735", "-98720295094103", "-3387455956043", "33422455841875", "72646014377284", "-54216173525379", "-107085335515558" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "33982846762129193", "18070113177751228", "-15344533360160850", "22967009312426432", "-27693105018949244", "-19447372654909311", "5526992817880686" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "22051072322444431", "25739994011608686", "17590147555692963", "-22084188037496810", "-10818514363912056", "3629814651214680", "-13864921149293605" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "1477505808877528725", "-7970299900821284128", "797529455893208185", "-4380462832684600", "-7512565283888184630", "5963752963276377801", "5383033152533304312" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-4687001791650362047", "6189713819111473364", "6157195100346646067", "-7354274856118962216", "-2096503882079843374", "-2989958959527548003", "-6990431314703212402" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "1543053224502515824700", "572944953559834661337", "-463017450276577476402", "-814539245715762117156", "2125610462348570304645", "-1998974164978877476896", "218777334538713492026" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-1737306797700142536355", "1531804891962626834702", "-1195046210647179987202", "2315927533477943693088", "1955617169829065459239", "-622025738336427455680", "1268127968239165761772" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-1605308004306756881486", "2011474158613556880412", "-399796168662135031326", "202881347434850907252", "2019703887423617334184", "716418631018448992152", "1206220294128077626983" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-439233385375980423579188", "-282388747616905814633943", "89357877650443882966242", "-141495542322748990965271", "394310253761853869007932", "408555292651658924207417", "31352288232225179050347" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "336925051011166695076987", "-104340070149410931907855", "-577270465951816735176538", "-257830029049203338174931", "-208291608961639617152387", "581220927173979115429157", "-486086206006317637743805" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-29078699886191649638085668", "-58403200017753206157268711", "-30728609198387149996274670", "-64955510531916364593021196", "34821390565272973390205800", "-93422825258783366802978133", "-32247179645343680253023813" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-77365678100681876623290641", "-86628826177603420327288998", "9220094682835113941854095", "93882093552015539929945204", "-150721224176965634213377599", "43753026435426938659269571", "66570450267194288255165042" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "26300179792340119797864079", "142433122880117613936146124", "-15440885853316070864672346", "-77502061475155985937091480", "-112852326686297479782561254", "72434505811059791598110246", "-7668392160648564895359533" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "29239766717040730171831641774", "-21707593058421917306282917041", "34516607235563142943063617602", "1193640916706101190985687069", "35195812678373567674988713830", "-4604316293532843686268059747", "11180352632046834701829653097" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "1512268160667408885922336144", "10844547914295271624025530713", "-36215819512055547591428396750", "-2810331321951135608956464976", "-4386752948121646037249312193", "-17767553783444186492160681129", "-3024939524547086439531257635" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-20864326575772010142787316454", "7488713151399696482410740417311", "6381276422733482520358034285380", "8884761350831459122090428512810", "9969195609900524295444588216806", "5543397900882544648824130766348", "2837938767703679300265023672696" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-1882854523229835368724322985294", "2966116522928193897186452665796", "6823275008349281974665560465394", "-8062354025481937215315488548524", "6765883098885847750323695485797", "2224258611402637372812075498606", "1096213966837685819229458326999" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "1594645032934746910233970766445", "-7888607865300585433234547082596", "4568621404199252787882732641526", "-8445702981648004651693271114722", "-7061451521435267187519509650848", "6696570366244357679289327894924", "8441199880199722664043107998393" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-968352259610893197064576275508718", "718654181183934094051419872500011", "2514405299754440481837754186402421", "1800516623297456299756852785790978", "1735762811642360734642777500928026", "-818427028473555572485098076829014", "-1159041509361012508054712578014787" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-53546281846072725871658001435887", "-784630734436542795241759169811923", "-718526749795888797678293952454287", "1156515964581299989003720410728362", "-838405070125273270332512952593048", "1940449904167314426051872957919891", "-68651512522935345206569837233327" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "572850491437435230154655178596404256", "255907180028216856698708583682335802", "467380937533420611084983609639927678", "-578776848219875222429940192462905503", "-305437010432528222408867961277657933", "444241914984458649589370317558151484", "-437642956538688234838513076311369227" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "358312056397600081928352831952683806", "230966521966579976309413468582413234", "-454572317314346054050950281223398637", "-514645527798405383965741984798628527", "178604228774099539826674396572478175", "-617509561026367858818205129274682551", "516749383280600313391592174692222877" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-123120621344099237933440205349374488066", "-32225408253558378935144372215319566228", "11301001788743914587399424830593135310", "-52673174720412201741335076711264329300", "-118712794710184498260756820285222988657", "28574446760716238823537127611396566250", "46215799970023783748448065488919483477" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "58353390412480195555770446835562760276", "4833546377833498062150899905495355127", "58373007706619991431855338114454320484", "-70703504770529836816766481390613100369", "143910284033581946098683712226489458302", "117235434241994039601754355029907027922", "100290083016626428361996754831114825823" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "77825035066067814510953781223725256637", "100986125463015864365041226255651190014", "119992813429982672016306816732813811809", "138695082079580523445519172685734788839", "-93300896559078568544280378088130535661", "-21512916011354134398283958037251369915", "-59254623236285041945205874576696705580" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "18754", "-17855", "-30143", "11781", "1051", "-18103", "3181", "30569", "-22565", "-27059" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 0, 0, 0, 0 ], "DATA": [ "-10966", "-15974", "20424", "21589", "-20157", "30323", "18376", "8005", "30511", "-8141" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-1843361", "-3517196", "4625468", "2321631", "-7687911", "2778344", "125068", "-4052771", "7945865", "6630921" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-2367184", "-4826645", "384943", "-2111816", "-1600561", "-570653", "-4103404", "743335", "-1107912", "-6354480" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "1229589203", "-140746934", "-1838286677", "-1878305406", "1961215412", "1250128901", "-1166900225", "275507073", "1193731503", "117609926" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-2112929173", "2144863319", "1992254245", "-1937627007", "1514668427", "-1839905388", "-1528455662", "684780309", "285489556", "1981384197" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-1886553183", "-442912398", "32808274", "-183696654", "-36971109", "1971012659", "-1278373247", "-1688683464", "1543369024", "-1978222221" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "516696037327", "-133356328060", "366887410082", "229447810393", "-446554774147", "-403087474672", "-332879324979", "-201414959734", "69326493096", "425678448609" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1, 1, 0, 0 ], "DATA": [ "-439293553812", "81933805742", "-183895960348", "-496786511195", "313438779807", "490045940301", "475167170007", "-102296474200", "-69324984155", "-419967280054" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-96551143452313", "122054609553289", "77940246274132", "58988438553627", "-37681781403480", "-19891162801465", "135285406649690", "-79327539900948", "132470331657036", "-6480637466609" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "139797814993535", "30274745342542", "112817172422674", "-52131523918633", "-137459606352282", "84222662933576", "-98419098319107", "6437228832991", "81243750066175", "-6421108664511" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 0, 0 ], "DATA": [ "77769134597368", "72127816622848", "16103132204416", "-7118137065494", "49038964068498", "86622864654107", "32928568991190", "-93632446337391", "72127053286297", "-103694778424640" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "22116565043461009", "-31711615551037511", "2821246731201391", "-13351247986191283", "-15623501709378161", "25262423813103532", "-22343569817384923", "25928473716871331", "32038496333487076", "-6890071520503891" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-16647068290977681", "-732120677335230", "-33532489576560866", "17792605822942940", "24198243709120098", "-23282344775422150", "-28675151281288448", "-14495501421676227", "-1863883724884560", "10312379020348421" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "5502360856538125985", "-32644019016852151", "1564692446235537384", "-117754471439160800", "-9013223642801616724", "-5023267401335837159", "699232645927291479", "-3584329191084872093", "-1311864165271950824", "-1215291485581331201" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-2526466500631342193", "-5409822186679340641", "971670265469264458", "-7179072119717361795", "7670806788162789754", "-8318882969144120324", "-9117830171517155584", "6877332662750891661", "1068609140965782417", "-4925546114463183284" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "57583945723378415772", "-342232111928136964405", "-1195237978858213658562", "-2068806789801380063600", "-345026535447781896266", "133610950746109877229", "-450656040906061504998", "-1893962609238091988400", "2350599054313986251061", "1880733516481080050212" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-2249590711027756564860", "452844566121110339696", "-1922925439694884164421", "-1550876878150832998572", "-1185260961134674742569", "-2039283323590053548044", "-1687563918950261572963", "-1944597896900829991591", "-2359460784165763037898", "-849961236635718175763" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "39964951314827779563", "-372589960613063031098", "-438035602871932031986", "-664149217874104492300", "-775029718832463979623", "-1435477218213206627730", "-777662353097751064825", "-262402700201170887022", "-1133936324748952947710", "1327754548066739872375" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-190787733056613599495219", "-515734754496362833319535", "-574734864992365444843708", "156606116703304701360073", "-548629563358608342499088", "314352279479464825255103", "244688836299970664478495", "223517627384801717925625", "317460265141706669756597", "-143088760391778698353724" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "-121682002183738855835622", "33829240294738771467442", "561539377338103121964304", "272589000480746001955982", "476056122948827385095616", "-483571374749297317898197", "281143130465809637777850", "-252495987618098543389302", "394527250215970829835810", "-371881433858127168330471" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-57554220355698316288465256", "-76017540520824653711945808", "-119283376705753351997748235", "135285600664968889113349822", "-56831347642867226553961594", "-16457633406117235070315269", "94381218510174633599528949", "139553957700687764923892613", "-53986781389720021194280790", "-19994198089108196976112296" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-82576387180835452318210228", "-49951144242867126451613703", "62645934474937602509471162", "-22809010733349138696349721", "92648271949016552757106864", "88129191312931452389255255", "48011438537493230295219061", "-78095235259711088591460034", "132652997750328176815451796", "-127305214162620317777045127" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-130726889859542614597966076", "-56150188916918748446918856", "131676166120048204530142114", "-94402515536541962124681830", "-141920082631622521770167745", "138796510001418272242292175", "-25121938667485895566695315", "-121172442589103978423419243", "-20614593640357879353182241", "-82772743389453364089604477" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-29032726342801534955983348833", "26158471499703236988192974887", "-20114116207465444344361601128", "-8681577131743649610696414703", "32981177147751632167653843307", "10715468167954459453141416666", "2302451135115289498612037956", "23070910964316738454734382371", "-33075125932954753049741121323", "10179913539917748846617161898" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1, 0, 0, 1 ], "DATA": [ "-18085869738221726864797815861", "-22337076672294140460410142760", "-36126257950244891489093166216", "32551071786984003673681244792", "-2850364559142517973517854327", "30988135871042625915127743204", "6816014999562698392234131960", "11494536675490631889014354130", "19518893252904531723609256491", "30427336543604275087551984403" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "2206367769081757187272650503139", "9091361631393367270835435652597", "-8424167241080231285345475239019", "5834959797812027161305666236164", "5192168644968070279507303594280", "8341506527326618523722473942428", "8594436209974939893446936907176", "8681682547098842826265765019748", "-5209020223570640459802520771750", "6129943703376515947722773721600" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "9047109249979470996987065428892", "-9799662424789913378125551587573", "-10747887838497246955184907970", "-7651592187822100343817178833722", "6010095747552236822560336347090", "-9204712857936864198407587203238", "9268329208382510346394692142454", "8372710454424791445966082256342", "-9589445085108256491425243830947", "-2939036102518584405497956526954" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-5392971967770555579833623091396", "7010238060604152385637944463012", "-952157695963924409916700180566", "9774556009383087545392285658966", "5770727225973437448322228561807", "8118034587530249294761621086593", "4629045275285811417199668511474", "-3061439446531807421688403151892", "-1370462548021359419092251720599", "3949264862782238848089672597134" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-2122708269622350729686602450849918", "133816358693598033876477261324945", "2416602162368187687173545193833207", "1033421043633734739603132153227481", "1940044255763482239251273519489903", "122404165677034509854998880971649", "2115709380653150711430911081473587", "-954697183455914334514087563443341", "-2123319112143807840315352640840117", "2069575612420264676616069495687633" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "1692523376605227556299724175506026", "-560133778627251677678012123636755", "1544300548641566325099709876685852", "745074432149870035759595939596681", "-1954945128503766900016994683433198", "-2250046900317020802445312714834438", "-2385830584607782067723440542099762", "132290783232525067349828502622213", "422538990938605269068919642797432", "-1355056714352703736683115064904273" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "569637448050074280085390862207666873", "-493782672094432637986628201979332547", "559997888965253767870152932047889365", "-192294596493342415461227479481558644", "-467823747943914593693180397447717393", "-180358592750289795012663508384651702", "-277633416434123648393250047500800405", "-161316286985680702736242426514896331", "-107565346718017079645076760979976254", "-343028720252730035978891475737130091" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "234202228460068979568708307032847949", "-40201170775316769341182286764965389", "-548980577113599607444799254971433151", "541309219470209179356416121398240002", "-434133022569940359270892323517092074", "-273560039796167418597819102027246382", "-476192949086818994497703488377408870", "538618952671800675393447490544116473", "93123165923308661421936492785506839", "-637742333921319974000307483554747580" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-57037547117361632743720819923766599836", "-98043608090325780586069692996244560497", "-58505160656452186950211173738911382547", "-101832799685426843314478830415915453714", "88565658613827585716212234902487873212", "37993662471162893207714631860591274024", "142774451425889543215285746860042086262", "16769512069264042461946104154625579535", "-47552828674887709372081650891476109271", "-42213620497501762639143979558467451605" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-147371756269307190381561519577768333146", "50001054768402174217442186921551216212", "-37909857998845694340361772993373171274", "26488762795056480696073949053739280517", "148945896988690949823595206483101743044", "72542903931524796290465216241047509873", "-167126191952176091620300581762459791416", "7988430434861579711046106333746313228", "-93130434033478424433799959885140345146", "-3441472474979605847594343058062446207" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ "46136426115595487614083206992753880226", "-152038433918307384545322896116023034965", "78938265199256609159965233504637641814", "-92829030003981682199112271693126981849", "10428026972710553277643628999246199628", "-733720183320183133584540075760187261", "-21944082237248471904000697107198113161", "77626504391158473624750083478122906448", "-159953023283206971997917750080649331297", "67741969038939169538736690551212738571" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-14965", "4396", "-28370", "13432", "-19302", "-25134", "-2325" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "17774", "12772", "4697", "6699", "10195", "7948", "-1073" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "2170666", "7768056", "1730086", "-575771", "6874141", "-2918802", "-4009750" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-1145774", "7588162", "2538941", "2814807", "6420688", "5552162", "61716" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "475579788", "-2110736469", "-1038349020", "860173314", "685087056", "2099281387", "274916519" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-1655873120", "-1731031947", "561894379", "867850098", "-1594780953", "467730702", "296381326" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 1 ], "DATA": [ "-270809066", "-1724753578", "1076930264", "1046992208", "-900063126", "453135840", "-1402444227" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "543756676973", "13894519068", "-471273054319", "-361667009872", "347117046185", "352117261872", "428679947623" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-414574595169", "536827742264", "73835628644", "274376152363", "294589390066", "-11991628898", "-253347644755" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-19215636889418", "119919740703605", "-53215288815658", "-33327361054276", "45198301326977", "90776449725838", "138054869379894" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-119367237365312", "-140091389309758", "15475687696688", "-38031515520518", "42841758149017", "69022243350084", "9179513583233" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "41132552781971", "41253830277347", "87359934843401", "-95181726796502", "27768710470810", "65809922501001", "-24863795602983" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "292236195126093", "22719593938564618", "-19218388071157930", "33981265825341884", "-18509717253371672", "30580146808867540", "-25999233818244727" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "28135088215434228", "24773690318810918", "-26724803220639117", "-29814061818363701", "-10766489482800560", "30758073781326745", "8776059520448382" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "409254481003117766", "-378161134041351348", "-226782297600641326", "-2884040847263852027", "-8247477639592596195", "4107975741440930492", "6909135551839426970" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "2652868014887595347", "-938854012354185866", "-1598192800799770751", "-8491269256089485178", "-872167601828525845", "8058502300329141671", "3061608648711594103" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "146769264186631567632", "731409928392690360238", "201264820200577960524", "-1212063678184800014575", "1402459273537500628832", "-1321984359334412491423", "2211457652738431111859" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-1847588624832707279911", "-1813942494389916835789", "-1958404120436196946126", "1723793223754906327044", "91104121605339071868", "-37127427643123176929", "-1759281127620720025265" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "621171459171060457840", "207620657443600214064", "-404297398702992699514", "1641661352402425721053", "-1968173283051339556925", "-1975771117070391869479", "-2326206316062007904468" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "157960797511886924834578", "-341648681181914186464072", "-496733650079984967364838", "-444089180371365298276212", "545885788394993954738089", "335428473992315516320770", "-8497165329921324825223" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-38994153758728840237480", "300952907927512214570952", "-312533402564800726680664", "-384239115396249647073708", "-93282765318781444842024", "551248528363972986694971", "287731359865140964525224" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ "101025247865168925900922830", "54857286231629651663501039", "53544859669524419312467353", "-124478605232939875253548620", "17285305918706892564565360", "-109106338103303620219905680", "-39866206645357083787763508" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-85724769944287781078609302", "-11704976431025150432522098", "-140709446005420876354105607", "118949605388441528679717522", "147415669254850277731131009", "-2518242221011392198813403", "-110299103632345032256326636" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "132459645720204436026597326", "-75684750458331130703402700", "122558708866408634795154753", "-110485976053300938316559281", "49084715923367850211269998", "96960123064705845023119238", "123180430855167057989302957" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-18936205707758319951200347669", "-37258565469174214424589588093", "-24453048320489113118458406056", "-10587746588216050364022206474", "9748825791372076261497902071", "36545682723514548638183652663", "-25861598120327052219989832099" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "34252212325207935596745602246", "-23818237350808788371236620972", "36226019636259967938400145050", "37192973754494387619137336106", "-5057174993081801990711137100", "19315470624682365399645492525", "-32772028697629595902245624670" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-6792023322456998237800922079479", "6549013706052367027552378139504", "-9355871158471909412376307831412", "-2030077667778674905770995357453", "4297878705058522858422532653176", "-2236484269757156229891142724479", "-6040135630828160347217599800148" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-3599439677962085785168307719800", "-4245900385327794216010405726579", "9450764155817872025990436327715", "8656233260540192979968625227977", "1160854863482601034702346632871", "3956026374789035457631650901065", "1453599630928365440553293992121" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "-8915771384062939324293344072095", "777633297413063771199232835099", "130337830279869604841710324103", "-8373921460052275293875653314023", "-5246842589420902120766262884279", "3729544269328940809924381217876", "6272763819900035423590366605426" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1617716673371462402162303323419703", "26336765354063275118084261528831", "1492712908830805398072707084234922", "-219159852068967847470786902528212", "126542964660097922190866932619843", "889878290610547866308346440500254", "2086140459688639708299971129281292" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "111351977899132479809610701899734", "883054552995427224030821200597311", "-2065448421761013752757262983625041", "1742123659066570365356027485580166", "-1214139633564246959114082011858323", "-1329417508775576485092806838462803", "1116415564152726849396179280195975" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "25792382598206920383056857113445849", "-225206309711067108325919531991796201", "404950774727900383585705746884848243", "-470637440392131155991087245263637994", "230147369983625933489964301980421634", "-371436034781333002283883627046974041", "560240201817281749776706447503520796" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-184751107296655537872267341147777926", "329426487040671280863523825102062539", "-418105543114253529014238351683881534", "-617932739175887659472533031137397189", "498336206481632051733278557649532891", "580985420014217396923072878340987450", "-205539611656251655979522315213206497" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-140752175559703681946897226667461340778", "6561236723375434393948178304575297047", "-51434996264522492401719681071804511089", "-125946647710302371061500790543557458955", "-133398205151504361614506341930758954997", "-25733469644094471567565487452545498665", "44048098477441865782455673692289495153" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-145448294825692173492641127474455264094", "-121372206006141296987868818880832869230", "-141786136125809700329851146916010465431", "-64699237500646340071065763833604265646", "-57587914054490682697769053906782102087", "32192058203820777323675957815817034772", "111541312639824620042046577015667034684" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-120964441901874362290104187440391939364", "-96084745151601772727177422680207591805", "-166536914660320577793788842820747499277", "61156293370756011979833006538029339427", "94985231961279433991237396849461866817", "43959734247759347414496739857098340817", "-72248466678766711563463040091419174562" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-6120", "7057", "-6295", "14831", "20732", "-7444", "29138", "89", "-31431", "-8017" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "32489", "16222", "-21842", "-24054", "-29957", "6345", "-3733", "-22295", "-1301", "10500" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-72740", "-2017731", "8013529", "8012822", "8352456", "6421813", "-2425529", "-6607883", "1881808", "-2234254" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "5181661", "-3577958", "-4493738", "-3295135", "2544246", "447427", "-4432217", "5249513", "-4448902", "-7992598" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-1569315361", "418335588", "341076905", "-730203548", "1269521259", "1784051884", "510427112", "-1268247751", "2018024071", "997087873" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-162075630", "1319926606", "169298867", "-921839919", "421705930", "1771873235", "437874475", "-312980929", "-392069627", "-1446413247" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-699761346", "-1098413742", "800470076", "-841985366", "1782178243", "-89008181", "-1616551429", "1946816503", "484187966", "-128499562" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-15033977299", "-446057719500", "481043288478", "205263251758", "-56863332253", "159382478645", "15497510247", "-11166860082", "459845389844", "-315779255031" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "13619924768", "-108585103579", "-286333703121", "-22225593305", "-4214443693", "404081003905", "-266115340439", "-522963986885", "-81541686728", "311449667067" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "3440291968557", "39790668036317", "121143735330951", "60300700594106", "-138565677223306", "26990651262138", "-69507089026735", "28346271500562", "-68434454958", "45308688118409" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-30985211143486", "14497903153172", "108091472626361", "-132443509279150", "-124710978475843", "-22621813043867", "43348381090099", "54224054718466", "43935291459066", "106409780408478" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-37426243585738", "-136731105244763", "132386170102737", "18729575753580", "-61381755765885", "-67683935577429", "-130715456782347", "135506741441431", "64115570795686", "23904178460900" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-3109726999519689", "1933126768037472", "-324967063575891", "-30747070522132113", "-27829185300215897", "-24226320094880357", "8149118499338522", "-14537608810060536", "-22181098580998350", "-1543766855453763" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-19154567959160505", "-33714007900076962", "29435912661346051", "35448248561967655", "27478015216841137", "-9642727074664549", "20762876950068092", "12674801929871187", "-29272042854662323", "23087590778346538" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-6178990937608128005", "-7489771860953428316", "7138996857544191473", "6891721369949758163", "-2873210693718606357", "-985023314566108394", "-8482975523737005038", "1933121339588994379", "-6237609370968336133", "-6006493915891200615" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "4784228838256112666", "-5639521520326372798", "-6372119935029533050", "-13173761350189796", "5256559774234064648", "-7639941297675590428", "8339054501857458814", "-5851965443287017683", "8471016463241511679", "241927931394313236" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-483218027063613629703", "2158782735740085418214", "2086327309425994289085", "-1775936067034501989571", "87437755811834396167", "-28401153096633450165", "-1133004021229840529093", "1871364394906942992468", "-1367249310926305189706", "2201573961610082633279" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-534665456095867317406", "-2091145637438156527744", "1002549198487665026269", "-1829053960401122456656", "-507210040486661453955", "1045481287999809175028", "1364780996043256384861", "136073480977825489240", "-1037248082018171019376", "186526137004977787352" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-1046613079302385912677", "1021357072861918022642", "-1212895629888241074297", "54602995841860468967", "-2328862578607448318042", "480770704064136552652", "-2205462302944763257608", "154486372431065433621", "-1627545550801203651319", "1661236305135671491366" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-25780050424235659153148", "529910266796805109246568", "-524035727043667850771484", "158037899710407903896118", "17775205387059778764118", "183414271106923495875624", "564101025301896167399950", "174082107953749604159417", "-510247241020792959694652", "-31790257572439119641150" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "97909310721294570420676", "-370685916895270859148094", "-207704549226119447779963", "426685027916926492394771", "-443922064163037127693132", "-359165394706192831593487", "232450412648501709276260", "236476482318617346453058", "-444855507997557301532066", "-449511488318489004842616" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "38297941459228367137032480", "37693160058585205753349341", "-135230435965114188640670954", "63954600949868348155966793", "-18059488984707719081771228", "12523259782961827464033972", "-154035891838969004456038899", "-143740984881393038015437656", "-297107082262287737774124", "-130063144978385075413421572" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-121901231315087756982129772", "21641765111158040359351177", "79510642319690549494318636", "99992072265161738668933440", "-127544045958283514926879895", "-114814479146472538283905870", "-6956674791158590081188973", "-152063567955740133520378519", "35283887938627339173013183", "-77488852854891430841283266" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-35297860946578727078961377", "60914664428732973645334926", "-45881788007565335539581505", "-16228932271534174997165412", "-71589554478597723028529083", "125241806056796930297219831", "-5330340415825203344097529", "51775257591241051145892674", "-15094772395361486242521560", "25938645275060872598806563" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-37399043524295253068535150516", "30416211250451302199476070853", "-33596129413262219560422312517", "5313931682258453540118628812", "21048003994659574916664477845", "-37382894423738432073629190847", "-15318095353333670892374528768", "9624056340871881017715059148", "-16108266238759019605768116174", "13607318011608209290042469481" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-29339095371120961510655959746", "34499077642097337163200996883", "12438610605314642729365206169", "30026217552330912860684967035", "5868323165870187705662338709", "-22530513597085742496802571404", "20187377513734548340151159804", "19309445557615337894124357368", "-9834900335171895983684326501", "10977735104041542369404969198" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-1375924286262127484011633937832", "-2147016509826786120906015229213", "8312572357614785735606012528886", "-4051638098169024644782191731949", "-3093628440576237730249167513209", "5666878520633789566902037223594", "8891965700337502827869504398273", "-3425704790604702995311162396482", "-6523722055221807095862112250242", "6355813979375858041213525598182" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-5058425412398335303061912485394", "-7219572559975239271635947264369", "726522491715344256514490058497", "-7042445872695173559471595541793", "2824402214773339173380186665274", "-5015784004233448712622655297019", "5486760745287225222007329633188", "10065222761695430251439328556933", "6166669897599573455003032087428", "-2172662097799258720481847032564" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "8036993285327716214363526967047", "4430074085127306659301038354391", "3362256777260202908602746872178", "-8231357790598764797150571723919", "-10113083855460126423773664315153", "4304195785667284780679333446327", "7529022456114985130352846237161", "-1617155253974148221187702205421", "8306989535252982450393679766069", "-9917879652040843830362880242798" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "2025150411878375809601981318007687", "733154270680517806621380045304125", "237851062444860227206988707561392", "-2559173073631870285694845698600192", "-1612871136126666613840893942772465", "-907529648416027223413166136860388", "-1839189396233732473729596583035690", "88713624503691341082574108422443", "338402803145217006589478543497297", "-880122644115442177811028160833635" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "1839742964889166195100300692473716", "885434974408265966958762326684680", "-1621828050026346614658514674603315", "1905665376759464813614892131583581", "-2388201929527636648329898098952877", "2481877047862950402685641670256262", "2023252640349615986505312617764433", "-1573486913273646294139059870810833", "1805414520297065811367995244619334", "1843061646556820707723637177021074" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-191847997784246847874745009499399054", "372786599802160285128273261461255881", "376288273710337416992985070127844806", "188093861376600388183605420152756308", "571207793825131270518817095820136067", "-29919619103217554294662107906751719", "-452678708024430178885781299576494992", "535406994206915746718046269323872661", "640280552739110271190723844875415792", "-600187294174532632129083155832394691" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "160335860551254932275470660660164261", "-471326848390488335045668822156269822", "-495691107470112053256769582261403309", "-393200617684151831876978692678316885", "297414113438652864360156678843959711", "-268776834626793693334448405382040859", "-117780897867754527460881147518546805", "298322417140430849049049944396274055", "234866336354615854416417135899865645", "48595083212054770623391355746687188" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "46767536875344963516160138900400417992", "140769783767864571592597846120527955449", "-87767956607006201530442382053058191437", "-147467086345147382218132338434959967344", "-63040799227207808871886672405579600857", "-163413759426610481573012906592956738336", "-51038824199236863589719906709018954970", "-144531515418551447921937889834182877513", "-167321975433722954448458398645759050579", "-123967516821302237595171386295592191011" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "79453031253319883075571062614534868736", "123921984663798339646770377207635911524", "-157303675968253574763215319217464671001", "-156891299361397755454566840636980983938", "-145862034846283521653632276945573288715", "-28726972658144915880841740075557050787", "-59175893681203716281306184534788761148", "-116003172142553199810126172180490708929", "76631143543919820921892254735365792596", "97582911312902206555664841122180441248" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 0, 0 ], "DATA": [ "7093132471014474767844887751733331003", "-108686417660834376788570147811161987220", "-163594850104237544086018972477724405791", "-2302501739484662175406128065151266870", "-97343550981288966396082969127785946405", "-96030927988302472038807926974869041673", "-154167090334610438942150064192357641402", "-18793807036998611053269951477295987184", "-70904740499999719165305256191226042599", "70653097581681099304462157018443053691" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "26050", "10037", "-10374", "-29490", "14331", "-16851", "-12187" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "13120", "30961", "18595", "11399", "9159", "-1887", "-30813" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-1464633", "220199", "2712771", "-7495838", "-7981890", "-4964125", "1595391" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-5853521", "4344274", "-7018099", "2293236", "-7612443", "-2167093", "-2845086" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "95875595", "-609963850", "896088118", "-1715535486", "-102098264", "1291776262", "-576308153" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-1236728557", "-1637854635", "-984551882", "795795740", "8028990", "901192316", "-1111344946" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "378575634", "-419525563", "1883074043", "-1173948468", "-1824740989", "-522033444", "-144793007" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1 ], "DATA": [ "370947827710", "-353447519869", "284585997441", "274833182439", "46832561565", "371244974672", "202518766395" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "218531362099", "125513046194", "203189347634", "-304680830682", "-463095454301", "-6527992404", "-65734664745" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "124491338796971", "-71234357373577", "-70640224281818", "-115120848140363", "46780581253253", "-5239172476241", "-84243714348014" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-43659843831402", "97675487713738", "-120205430205763", "55637124317579", "-2982496726764", "-52205341209619", "61907219984235" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "40565389677572", "-115932875026862", "-59257993113702", "45346953112251", "48999715544038", "-93859032164586", "-82647138297887" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-31698638660098966", "-5631854766304352", "17449252429242489", "28180900120216510", "5008123877732808", "-34000957537070508", "-9375521335589905" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "9961273701300998", "-2079937397053597", "26948861976924314", "-33274483408865913", "30025685782393962", "23738162907963992", "493130406875726" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-1979493224765362120", "2703732660426547803", "-1393112844070577191", "-4589313647595954574", "-6650779199917767681", "4898222384404144581", "136147557489562317" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-3646410955003742974", "8191219785186527321", "1272327194703892541", "7200053761957267992", "-4756848297671371938", "465404651157646586", "5030791210523335471" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "471782014697430155157", "-737522866566807476021", "-1150886932732683487297", "2216993989609573889751", "1791930017506828908157", "-844823775260624667509", "544085813983975919632" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "1674397379408819628225", "-916247777654751098512", "770173233696709315682", "740475606804091311338", "1832098966387094053802", "-846509827577616730614", "1310459528272118181618" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "74011478604212791529", "-331788012397048455114", "-1505535048061915354043", "2196671714385448598750", "-350496961296855157879", "-1047520580803370592747", "1407733904504194944452" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 0 ], "DATA": [ "334057475536999391388584", "167091731454929333593163", "44371453303518972747966", "569151268328963633065480", "502857981392722544973772", "-295294362379075981585508", "231490385988114470344787" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "67874870060011234362111", "-134871563655837319141577", "-165820628960372861946363", "-391238480407275084709510", "-514577690826104710765865", "-67979890712897414695858", "-408586208764886574226242" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-92558993866389986507056248", "141516883852752944766615012", "-91570642406924423409457354", "142445780080422570580978556", "6629554290181300402543099", "-13376427830750013064535819", "-128889999820989497872482505" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "109570534710933797396607895", "44325882156245683484525132", "-78370164483658463265616463", "21269267048775656453724437", "-30434466531056204756754318", "-146721097875248361900368890", "16341220289441428332578520" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "149460511934390160021660960", "85813150145346742579054179", "10395595941021601123326187", "60742333060570890451324525", "-69744724753060176636143795", "85805585820369366977540699", "-118751782594197400635380486" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "25165154960101426243162674534", "-10828793959658115401846998769", "-15232188130968548112534724088", "37859448133639721078872362278", "-7447685266469730873213326336", "-6630414361379515759165805767", "-7233008330319849805678054088" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1 ], "DATA": [ "31352291163482491119505968125", "30926463871793539317337614369", "-26565368808580343750512847154", "-24547829992985946652408320126", "-18355556958734667364187798032", "1438511591107866365310462118", "-28634765763897394391222662353" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "5950176580747626568664247515800", "-8881211795988992975636965132300", "-2313319268134714731343698733583", "-9280914263091091040890831012745", "4639580868256294580236676918210", "-9335839789223791032443503646374", "-972698824035541842758234702650" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-9116871202867186426233180444277", "1252769228624504890201406664824", "-4441678277221598037171990272524", "-3497493513724645633286134222906", "1362086037022057987770970014654", "2526310726561592465837221454378", "197133844356911321653053339758" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-6708417134328302521535165500528", "1259667415597434897205018980503", "-8667405068669984535223184571634", "-5739153439279994263124834718486", "-4659094842072120505966726511778", "8300992321238715729975176338668", "8305762879837071470447907044080" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "1855360572508690069751614139995240", "913127495556918482781605103352223", "-1634237774328458406341648362825579", "2012213887653411326935188001113650", "1854844808900330191204660335054080", "1943836864945319401575398852606071", "-2512140816283887752192691293590826" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "2347346757752941921138702653567730", "2471720490360303419055064438101411", "-637525221921571876383204743520988", "-313171852811679112466522221783714", "1342661303952287975125536927727615", "2299702375882500241106744128006573", "-1943268622964000029081987879045476" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "654404760071118946096668775855737838", "-134614183491363562333301742647543553", "193373068724594777442105443158226074", "82128803304304898958437601302688987", "231616933503852269212186042174117433", "314512202435083245896423396120681411", "-560893487306878937612307892677922361" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "61682174231144284695206586082327571", "-129722213375102804822534176471924759", "-470843635185932669298551792606832749", "217981712620797098953060036718615794", "524439585175580833405205537825140874", "-195520289865208736968677273971035370", "74511399356182107059807662782310402" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "26120113210988095907759127757085647952", "-62165437783621924594919739022258953866", "104747421037847971642972590567045904195", "53605601765747459417883834171905176717", "-16460101511938648810934533084724073149", "87326150749852540273294368718992775275", "16683834934928424633510098174197548524" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "26497149875196872156784837245492823091", "92947688966177192205895235501995800904", "-128897447396708280338335738279922653247", "94341806002152925510991334018527791058", "-112437238136605911451884994064234664417", "-149010327216271969260130899539508200431", "-32345160422388704273102616927971726501" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-166189853516680988544038633926086071136", "-51412708930825095905795826687352289630", "-8345130886957441238365575658116421520", "1835330936447412638754115271624712637", "149041866461530306889089342295575456556", "16212245285186773325597991037224844276", "-44514833992738879886424708523206514266" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "19322", "19888", "-20267", "-32519", "-22546", "19491", "-12006", "24475", "7982", "-4810" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-32156", "-17491", "30836", "18967", "26344", "4302", "21070", "-3741", "21984", "20912" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "7093261", "-5853548", "7803379", "-2652515", "4959963", "8087528", "1598575", "7487372", "-3584523", "476277" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "5614990", "1605944", "2158854", "-5461816", "1168967", "61828", "5746557", "1413562", "-7871958", "6056696" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "1038435346", "-2012132702", "-903699578", "-1322624637", "-895719624", "1176645993", "-1538792708", "-499541589", "-1303395798", "1511778212" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "592833317", "-1551950984", "1787888535", "-810984372", "-1974703475", "493667730", "-1121250454", "376054319", "567834251", "1569097131" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-57685736", "123800228", "519450088", "1079630019", "-1254308991", "590805934", "1765339826", "-1234550696", "-824160185", "-2094981215" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "426101820801", "16436911678", "-359476250071", "-443809896688", "320952321382", "23645867134", "341575522947", "-454866134852", "140063572947", "-548803465692" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-206275629357", "-54167785313", "468398637611", "-162195244968", "238463244086", "-168579433699", "32497103886", "379346160025", "541309413663", "187332535600" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "49075412231132", "27312750405736", "-22512416476092", "112383900203683", "-102429519587353", "49012285726128", "49380517981528", "100096915120043", "-135774832908052", "-26743412282490" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "92838931686727", "1117744174578", "95749441954752", "83774056709063", "-80485560556004", "31410756920117", "-36487657625110", "-17337070785087", "-79624495118450", "125937356449490" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-110960987245002", "-130196863138687", "112521051717937", "22370815190676", "31270809959205", "121855313393173", "8699496453047", "-91463041032135", "120519868366012", "-133257403196016" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-32069587070878257", "-17919111411953962", "-18001660931442915", "-1767754592277008", "20250526488187012", "-15695508630258145", "-29837740769794226", "-4501074155399095", "12225232288612758", "22313124950526297" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-34480102097926967", "6344112291297418", "29002029974785001", "-31888655934628218", "9139086331212235", "2897750950926006", "-27160809101795530", "21028873622945161", "15244977183310528", "-24528972829842178" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-4068321997152171044", "-4123931468795332282", "-1917336520397580474", "7863490469535036896", "-6249110314124783127", "-8225844925803348664", "915179017540453539", "3123520133722242387", "7605858146072323334", "3943872340866871094" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "8702547793259894978", "6483962839585495096", "-6872654786944293818", "6033338727343656838", "2377337619842982990", "-6489810207155348116", "1074629160217093102", "4824865193186455416", "-8000377732360383694", "-7092995275501204023" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "392214659868220760435", "2001707758785583846477", "1973056242078770230918", "-1958109206556224024652", "1085003398984190910582", "122474924730748113507", "851293406060406798668", "1832853642649160529635", "-2111330458370422801057", "-2122277960278850970588" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-2252221810790127710310", "1477205316920429874421", "-961506947433404737273", "-245308458478085909173", "357740715501747274650", "-319267889025708102500", "1094364280480774943760", "-127746993279336115173", "1782157345845552907164", "238471321966246221649" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-1691113452288155441823", "-837928306525549514253", "-1479432108576007046222", "1105760639244233719583", "-1237878533885748567381", "-2185454825581323446807", "163923040453210887447", "-1194656668349891851539", "-1697451474233505816259", "-1346443165535100297337" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "234606311866196287199252", "-280339717607805536038094", "174162930698908947490765", "161585530398926918641746", "149110074266065657434558", "39179047469267699561279", "-562131507501967025468563", "353803686657854340422174", "491584079078747706104003", "-361063318154572608483058" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "271411052426415377865185", "-444592158660720424250257", "-255709913155980939726276", "365550349570545751906209", "-391044572823762759576713", "183367216650532855576691", "-25390552937951110044900", "184949987773880185014120", "-267534620027159821434853", "132847257761444427988262" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-50975717540435075836318695", "1378903033683610665195435", "-140258399598530936546358654", "-87594983716718223901386738", "-62237980291352487006786500", "-91499845570297449212552189", "-66208718275179977437401191", "71189572681527467501510610", "-1306164209002345386773606", "-49463313801741626591705131" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-128773881103768533353175446", "-92989705198857589828906933", "135169579794423165966163763", "-74770130369838071149350912", "66417301185305992075241814", "-141076205408915017047534288", "139087458245468156248483059", "-113662293865384133240833450", "-33825577590872840313241969", "-14077765969840480002036401" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "83003901123187485851103193", "-2681382205142334385409300", "-63331882652492224187298286", "-110680165287453809004856775", "147522995254483971982329487", "75382769928196858065260409", "68276261988199153282208869", "-87269314773681476775194807", "51406486149759545599895079", "40564670281024197061922198" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-26644049411043351785728824141", "-24105812251123083991288681115", "-13313117369621013301443559549", "-15343445014815373573159127244", "5755441206841059236617659163", "-19137609460361732913808417597", "9820328360260799657364770926", "33319966440642342193193092160", "-2450510718121406857997296905", "19633470714253525250579882639" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-36841392181436281670139286742", "35355431815928332051589081557", "12542032737254605451781545802", "-10487948400679804251659176662", "33809232792580693385020804592", "-1286410851551344895362223329", "7299687240575999746351638266", "-17358024343798554471827987834", "-29156466412538498088209136035", "-17589710144892474177196649606" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-1569098507666310750400243811856", "-758234951593444817309034952464", "-9334359694680896788581614109081", "1374116640047423155668561508442", "-173162738574107411965825187244", "-7788646219993207807828069988222", "4582300712366442004380627595004", "4100619504434533691239280199560", "-468263237010220777440412988226", "-6978567372677615209814697240488" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-2976231597733439456905633512789", "5127694050127805803249796769468", "2812957412066585231215442690760", "-8402845131237017711805113621227", "-3787750299715284207846037832827", "-2942880408698349918786267176602", "-3869882757561265792893437687111", "-5705588172653590518606599812141", "-7302962533971552771835642777347", "-3268746639752527561077535734497" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "249971857312161074689173876267", "-242404619148151547781226427874", "9581515616266968745257306192718", "-9310338506009408582538577250155", "-6804608278893940697882201407664", "3099137104427329585255879001438", "-7567579129514718223375916274942", "-1792499874699902261296010365098", "1639591688226140840628852768785", "-7608440910939533514456193631459" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-1478066879273161886647445158949772", "1837220622062178767494109366244231", "-92877272254401719687763113724897", "-294736309677963931197700276380980", "-1131638498094855104492084200292876", "-1707583373250833555270111202726089", "2530465654746538337722203391829038", "-2249523431616310451983315854427522", "626377764251003872328656072958391", "-2410247195955462616836551577048770" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-146608320296582083230297319921561", "-2472542853509176412554981102318147", "-771312358883127333171922117508275", "-1484063485715773612247024703348500", "679091726809387681196416537626957", "-2001457836812405622324166631537924", "-631900342171956815934243043067448", "711088708081508591363639203425914", "1216331827955829693179876936054302", "-1299482886169500029305135186407016" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "200012218527021640529683894748995748", "66764908937162351505130243349347892", "524610465017564034853035195648393051", "-558690033622104772858687622389660622", "-544523595431390359103420871965400054", "245517776515380942196684249862925365", "212945120221746468714378114960847869", "-170782019116161816032185050362150848", "-496641648487716226900749343652261406", "648453852192358064522416946780906606" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "255949560736548808004852239641213264", "-324087200268712128351855136105640947", "-6820351909324143280484118195131375", "-319504958870141568923172587852075104", "523488920375771830477498953239602567", "194332300193562216166276513041179721", "-544008549012482630748073246202260640", "159071387785195543622408425447358845", "76192843992111999275453592662491096", "-473054289942710741781585231767065104" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-139359679896732931766919659337245494846", "-35545345721518693494350745566292372621", "-39327020288096966885458128121852385736", "7493284389294749331633573014767224917", "77381066407576226608915901862097770212", "157711333385356156585898944278021995524", "49883214513209304705821748055646582719", "146274344215989941722162867924721361448", "29434461275514518797058993245422008579", "129948671740191770106010881205508710601" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "78106395943978345433881498304008994451", "-160930953760985927349252065503891837878", "-44960294156570777230598296792605054671", "140984650860694548582383830944572104234", "-139997530771990989911328151859950982105", "-56309689550359535719606358796078640634", "56779686305520783893926725471749235781", "64518970089533247978457584756917625030", "-119002062601850172159773604273618695770", "99267635189640639519242595654851971448" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-72992812885182321071012171435152271739", "-55130623505249212822546937141683067554", "-72150173497072821802434510525219140741", "-135826146169714584749451385037745056378", "130698346424652975123966744269457918140", "-9260092987630855868815559991510953109", "687286835945665138009166453051890054", "-152468467896817629814259003076281203489", "-162737426598589027675694265384434548482", "130527505687529038013531520307450723070" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0 ], "DATA": [ "-20673", "4891", "21555", "-23205", "24226", "-28935", "-9747" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-7505", "4860", "-25545", "20070", "25803", "-29488", "25570" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "1487976", "1762770", "-4729573", "1860064", "4929956", "5522284", "6549685" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "4049960", "-970587", "-1321157", "-3312445", "2703511", "6707801", "-5572805" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-1030375303", "850853352", "1617347935", "-420547740", "653990365", "353737175", "887141994" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "974530540", "1286390864", "-1834371078", "-873048386", "-1662709861", "-1479216424", "841702675" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "2111707600", "-254659598", "486623667", "714212951", "1069946182", "1654147123", "818253616" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "158613753089", "458774753508", "183875050170", "-405459377062", "-76041901937", "422167883413", "-534535030458" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-189858741056", "-350281980076", "75626581117", "-124609040589", "16597068948", "210894043846", "147277886747" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-138751689194121", "-103042931779161", "-112178692829012", "29511789263726", "46232439685971", "48353150098534", "117568446080604" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-61910798890980", "-140506887267184", "112641714944989", "3355252674878", "-15180008486174", "-77620968921810", "88635255157246" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1 ], "DATA": [ "54206304886847", "-101630474357009", "96590747853874", "-118603139384744", "5675249069187", "-45370506866973", "-36853313570962" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "12547998279054852", "32079636436024400", "-8837895237364730", "-35089444786916540", "11941345258052284", "11998082458470934", "9244690049555110" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "10936921703728198", "-21359025539153981", "-17919755145627767", "21949453057332109", "31577422472892467", "-2175882775298387", "22366796586949547" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-1991785497370767870", "6209395018704963569", "5802436034677451710", "-7064032955625756220", "-425594384187662050", "-5892871677969375177", "6618055914134267928" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-9127990150600728413", "-5735750909960740797", "3801973574837623883", "-5415111795522083403", "1905105625346053106", "6231685532968733711", "-467857863906217121" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ "85863289349977744466", "1143903301173917583596", "343013108718986345006", "143977393722524587619", "-331683688552855746072", "-1255008151495427109419", "-367830894824428134772" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-1380262755120611917815", "-2015184270487688482207", "421650876988531672681", "1400851929839245970743", "-247535080568189823163", "-2113957207159701869965", "661753800403331824022" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-1409117742382861999337", "1964560064468429764667", "1142331396214943706208", "2120139614999229044568", "1947051906742118331879", "-2255925732391731193539", "-706486902189772690761" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "476431701466362805331100", "-102676717243207894859018", "594838701556832564313786", "-362093575874816415536617", "-298506140329613600409281", "113749430332489269875681", "197169454728417077892949" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-427853124038909455080219", "66701743587659683001174", "-2597511976969857899394", "60778757968804042956061", "-544547149575175129004139", "-301565858845410225401166", "-265399533535019605844097" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-68618783730059765480823635", "-148575561591978834067415296", "34414152208973548229001993", "-13643309285265500492215602", "102647265825215089997130781", "137227738860777958998527556", "89495140415736266671994902" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-136993255643844109804321368", "95521822621879235917169875", "15444326698047673705527386", "66587355931325005334848104", "-145548966688252050709592565", "-114049305396707161765587204", "-108330685803294296438944164" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-35055644549667511602728701", "-151408885763406254266448065", "148986972316531200205118208", "-41264915535541021357739976", "136259859352059633203136439", "50887898641106552333186126", "-125285378380785532529599731" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "12941490015157247768713837736", "6835075928133812117006091940", "-22465188887333363863609842193", "-7126648403394390900967730280", "-28393957274203526260439553432", "-38741812277298816415321041919", "13338414913605328772627511898" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-20110672505237420917225518191", "-27596411499885490649297106626", "-12603225725813579881802031040", "7543351889165927098806258732", "-36704747017636431900889798792", "21396851540234206754769905295", "-26912455145404876756203862275" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-3387707576912848734654159629359", "7428755755160883095771560631147", "4569162969402491573365786468348", "-2490525292701388792253454545621", "2994837362157224884748366006872", "8916264544517216999577433705932", "5151024374664908204375853841885" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-5663811480390229830172653604467", "7710539150230289621751006180128", "7797728575196483778606701813486", "9630462872777850496307059381475", "-7728740627143181530379709906473", "-7781087371673710496980289208707", "-343446002946672747378655962906" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-7557416815553078625043545761338", "-2302505358387623502188356176192", "8409779146217467126417839702826", "-9562089631495606093360342875300", "-8213115909769479205332628941563", "-3559170898222778482528552837035", "515283880696723945344198400802" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "1367641768492784058807182267663971", "465298046667559350852539820375203", "-574478792401028873492458572224839", "671454055256727078424602503807202", "-1606532935067250255205206946799560", "2502175580278378863196680582577687", "-158821802300435835463112943740860" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "215412964590856924411049997929445", "2425019052313712110758162804934291", "-2146281446953297190248965140031957", "2313116073547801117262414478722805", "2445004820402073310655244369464935", "2131568767414300517904191007403641", "-1647531519212921581334819756802357" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "85439167121228840311602468684207718", "81462378211070190176464268885238633", "-520261127199435865904944398789158800", "-373378915703658263067219343884833015", "303273353020035808553785984579294122", "-14150871600103744151748032687048184", "187409089415128244172342086722110778" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "426903599253969182311652501209225081", "386418676761755887855863585696427033", "-12458025011532389171806041982715946", "209536513458228602712253512458307460", "-606973411546439030075456065531198766", "292324957531014750417545379298738712", "-283787856374051062869985486340352964" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "63865640843944075030209963280445799369", "-18593804610606078796694478405978654849", "-714714379007990044568901737313354522", "-143842961516453470477418544067319181336", "153172084381164114541423265053189318206", "12685456738195001579207153151789306138", "-29575308771159264213086650783552893870" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "46883541681382683193881459486746081613", "-125705053509498712301219791891009785852", "62669131077706452018756304286321569553", "22514282939897237627731060578196173718", "-75575161713087697602988263826550011240", "46505563659899129005157792634893169283", "-119258175090032575441578768164878731653" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-55040330949772143040530849272221593209", "135813211061156674505901218739644432898", "169896929526665904477525636205199802874", "-144923747172463589788289511293391386010", "52887701532943518748533315738175210189", "129347148437659340779376391685263512814", "-119470842876568878095060418047486776106" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "20439", "22663", "-27261", "23271", "-18990", "15944", "-28565", "24963", "30418", "-30608" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 1, 0, 0, 1 ], "DATA": [ "-22057", "3217", "10476", "15694", "-6027", "-15715", "-23007", "-31432", "-4516", "3674" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-7442230", "2338346", "-2738445", "-5985510", "2142214", "52299", "-8166525", "4690970", "-5712690", "-5048341" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "4260441", "4125202", "2115968", "-5649222", "-3760260", "-4042562", "-2350779", "96900", "346378", "4216085" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "1097374735", "-1284838828", "1061029847", "1269187264", "2093057513", "-552929762", "-1669807006", "48971987", "-806857042", "-1203491271" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "486603994", "-1677973283", "-925436252", "1756729478", "2007578960", "841852782", "291979431", "1252234044", "-1077400305", "-2004190290" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-2012058775", "567356812", "1905525934", "-1504705027", "-66129469", "-562262896", "732024217", "-1858174448", "-1128956516", "1788342603" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "435657141702", "-478490014750", "360566647940", "-451660074249", "-473020751145", "392377661939", "-482632860063", "-338086490878", "309007405872", "-353813708003" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-459773526894", "-430668239377", "98090610437", "75247578430", "-324388649864", "416582345348", "-62640371660", "-284829666000", "381139880955", "-4984998478" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-20005911336447", "72447015480327", "45875595264956", "-76521915606081", "-34727867953542", "-137384660844194", "52697015634263", "103452307321328", "-122001791486213", "-41151199613229" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "70939096990350", "-88819773553707", "-56139660451437", "65674495025011", "26628603487458", "-135902172047580", "-103682776666204", "132777414234749", "-95404715253975", "132858731621408" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-132374262406977", "-23628151618575", "-119413346104562", "-55036741341427", "-35811108191614", "-1926885375842", "-45365158169918", "-114661135140377", "10473703483140", "-74303850742113" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "25306738711824109", "32509922510474623", "707085602821514", "9763707224974397", "-11143100390263700", "11596259349396834", "24075139416501450", "8655375690994432", "-18063068128417348", "-19863328717071201" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "5637539114033423", "-5001319200678838", "14212933040227288", "-33982239913110551", "9485454238704206", "22215509542635843", "-32953189204876032", "-24998747694223021", "-2633537128076828", "20245361874996758" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "4985469615170125407", "3351154357521105423", "-1334855791139741442", "-8797809572977779612", "-8598556274215160807", "-3639608199332585325", "4875792833346596793", "-180163167139705625", "476427630399101153", "-2608044133530342885" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-5157228090470659511", "7212202656918885732", "1392041261546235241", "6666890125347683", "-3802346357954570034", "7105488401526240167", "-4672914031095531393", "-6827233405808698926", "3167057671289501060", "-3717235008913756239" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "1189803804746023930692", "-625825002864892544122", "-1629596920467579686358", "1736386230972209967360", "700381856353328623786", "-654232777049969789030", "-74724090947393824084", "1167695718621359342916", "982240876673049294819", "-2253964136358443531770" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "1734021379805203643295", "2025823067259793494492", "-1597926848173583043168", "2016223048238280625792", "542138962133905683759", "879305494437540350029", "24106321453602867849", "-23115421710948790743", "-1394212641438626790639", "900304908524327153009" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-815922170546921752952", "2345528514985286552061", "-730780927736831533616", "1069468118213157740893", "-1997524152172607320568", "-1013146875657159489822", "-1973113169762410936751", "1200927779840472350073", "-1766824844262595417616", "525685124341004293754" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-170913940873623401775282", "179981267419149838084057", "-370082002746130953305180", "-131843472762558045740043", "-552464426412919975259597", "24312389763321025014440", "-117006219956015698142037", "-294656127901854426623108", "211409385267588640410423", "-401701449753764845530816" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "261336340980586457175758", "-200294377526284545507585", "-545096901807334905127689", "374297611040715654546064", "242023592078112962668392", "519439377841947434686615", "-474543347988347187804194", "-394392300302764104998174", "-373120785523247440518771", "588720790387528574187381" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-139459348306311875888174515", "31473852940256986171518891", "-9887972468444270565467209", "93418597022785737089002364", "65646938038849350878907951", "-13454108861792806928065988", "-49131526937539933217982757", "133308858070906575237327213", "152947960978969273942320012", "-99442640752857756581864595" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "2115383285040758640199104", "73985143764269122713862311", "61649530092631838157886499", "8213049092689795695398840", "-80077933562147893493089163", "78551679595459404249914332", "19850268736295064479696418", "124684888502488128271726633", "37465268414151592422174391", "-81920045880589413421302411" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "146919906028535181219026866", "-89543083067520077646503602", "-57021118916205567449809506", "123915826284653894642187831", "45309385836027080396265975", "108158825422372930192837799", "66329707408865417911755413", "-118958587438073608674047710", "-91357950068726260281104280", "-29334802080406579764325519" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "29971765432798246511193470890", "-14358861096333026316517453356", "-20738805649133754102834682386", "7189350037292179268626580107", "28348942101195751039833924545", "23861168485253092769676627300", "16588967227968379293949586116", "-6873327380426872277945057200", "10424009656429165482189206264", "-25621000662753343435110968263" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-39356814765782174445810187495", "-5109629353343354164045003706", "-16869835388192806041216992477", "-37493550780440739680416813425", "14541598222000814429300992064", "-38988737188675992487467355111", "-5094561545550859820454915644", "-12905499536852970749799435474", "-21653278306908362352193117144", "-993708667489234042402387802" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "92415685774358925610111866673", "9229208403038084890901114521746", "-1766812394389966971613945075397", "-414903185248431735357057732437", "128208901508290284695665875545", "-2137916455945877871317699306510", "533853066501738976626621316273", "-8107875934613418676045246063445", "-2240122730023917069540851608907", "-581801766541597034700720337725" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1815876156537294905389859319649", "-4726996253616925415079686314941", "568839304816351244324261819932", "-5163478890877212498625611240707", "-662774202918173654109667847074", "-475889796722743921788914957564", "-2257429926736080379736282434990", "1726653147329710337174488734580", "-9363121838594629328777547224810", "-7533717992682225954740846518262" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-943679878919633167252094145878", "8714088656079526494903956041252", "8411215846063249276308781788948", "3664164290851654853119485847374", "-6596298511313995187171485387486", "-1888023965534996939643588527874", "9495642065507897069829420814146", "-944088385505063048706519524817", "-1682255994436138829648213443423", "-10107759010048745798990652968169" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-1264118898522834382835880961293103", "175335396147539366747199378984306", "-1480903532901286343978092781681499", "-1192391017717251896100729091650720", "1838460835508247778407301651152350", "-218273162958334959288494722385785", "-266094035120591056222534225926678", "2529452524986175024854639128506489", "809296180497156248692989775063626", "-2062841016560545591440823200686963" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "2170563804277303232741531700370893", "-748753118748469760570998958561667", "916018945705575050717700426827152", "-2184472322228724823602690942023783", "931638193487785096171421593175885", "2048640881231787711057765873886397", "-1717558044075129776250633892902155", "-2313071463687411409263290533428648", "-2151217590832164263816790880109487", "233680104831469375662508975501380" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "386287962215958396998754574574732663", "-631240026268303131318958821819142960", "-525325746870165944407752164089651120", "226083291950635135851013581626817238", "-248862800389803494487190616128802768", "-504010349662551275225205203827997278", "105517119082759430552188298694313003", "485997620144390375276743453768158813", "-31031956533367407083197254732570524", "105470167017873351672604929878914316" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-483310365266775246988333151327302729", "632542156756981952900968647179691109", "-199686287544647062245243545386295768", "-346768888815915145525346260667844925", "-208917751619615348174629828212087836", "301058117385654246394042436715253681", "72505593750960021178904279470923063", "-533222490246855780568633480920745903", "218758514092273334444751656476699625", "508751967807589006206145598647069503" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-42741017306389443150668859837924739566", "5211098457938490686418682329098486838", "-129969844595428022866869746117998789324", "-94012368654033828127179346577805729097", "35086650888287131920843468969962483767", "-64547523416114956219242396283441262524", "-78626332557361945510915241951325611430", "130701192029686880255650902394824091256", "12989363125669154968811991286890110074", "-19085080263288132794957946754147030146" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "126277014750196607230171475233679635500", "-134087443068240928447419624897745395408", "85434520441165433374732053355016779518", "61561576435107610545214363658645070358", "-35737267355118855527515802602250082633", "11868126818381274069679858879103809834", "87005374759848073229912677602906854442", "90719075388093079581459445478934240458", "-21323038801308634493038758759215276208", "150665237002876200779610299709080110866" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-100088170739481113599978032714704061913", "87230434305877189656761352602340853350", "150644562584291617352733319990979317142", "2355696344715547537955189102551329553", "158450070722937152732551704806734037987", "148116003453147564228147700395660475174", "-115716266063960239779150802300625543144", "-158188859732442401491344398356371993578", "-99767424729140507618824095329194502037", "62003921486566133988081889333737019283" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-535", "-25514", "12848", "27508", "-25029", "-8130", "-25697" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "21948", "-6803", "12985", "-11290", "21862", "-7890", "7067" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0 ], "DATA": [ "-7148303", "-4971329", "-7052595", "4735130", "5847325", "-617961", "4747637" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-5671537", "4122273", "5239819", "5517428", "7254910", "75359", "2582957" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "1425327362", "-959771469", "934816401", "1185248203", "-959453286", "392425494", "16338144" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-1025370776", "-572097770", "1425619841", "-380090147", "-1914141867", "-910187147", "-1412977686" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-434340667", "-1848805050", "743213334", "-324842742", "292670937", "-1900640639", "-933474826" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "239743079568", "-159475972033", "-482663945868", "482237623461", "-423374537953", "-467862671850", "-529681543215" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-9094038950", "163457626005", "-104843767567", "-374539130257", "-470191859293", "80376556424", "384398483083" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-1769074858487", "83312732731055", "-41445089577043", "-93201479369899", "-119343260700090", "-68212756586620", "-42877401031196" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "133242907747130", "50022504118929", "2658782627429", "10941894796686", "-80085216865476", "71240008008818", "42520497691986" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "82862936532274", "-110751109078729", "-88636454965649", "89575893784635", "-113792597216820", "16670045303954", "72512396633875" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-18708696209667035", "-29729465986356868", "7302038187080545", "24647819387887401", "-24274097067646472", "8981835671516785", "23227166627732592" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "15050466481240323", "-24020755744281234", "30234678633583819", "-32972592121664681", "-31358688408363897", "14381538054568157", "-21711802576304571" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-2980656018277684855", "2130261546356574299", "-408075321227727159", "8286713698598987178", "2810785916900593847", "4658344625420734801", "-85955245053273680" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-5045733282821378120", "-5160561427270835799", "7274527635644960075", "7450048726897605750", "1192508141799813218", "-6935690825612507890", "4890840298456771251" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-2095147600329911271060", "1933472672626912165692", "-1347315152343358826298", "277387945491924944621", "-769378651588853688675", "1228702994603312892693", "-573226298560349349125" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "949955811513626444069", "2348831874784760844285", "-1005596983583704859249", "-160987643119919574079", "-226311796781453099711", "-777866278163013786942", "-119808887292662530484" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-1078706430442981789252", "-830548988474988813900", "252173722280322777802", "686764913963945110534", "-300321502707885602741", "-2341175459570789936839", "1104794651616574646405" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 0 ], "DATA": [ "81847995433813126101147", "-122374233040857342961394", "-86035995805787828514539", "126804988483725019773355", "173993212271803349346646", "-479430025014835876660217", "-107472624962357365868332" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-394077530826954690702589", "-558490301095757507242556", "-521490538576210425997803", "457192467034944298375289", "-514616069321096954519829", "478488130853503189910654", "-577821496311805716252063" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-25044801706278642778701324", "12810834987050168365242160", "31708912178391271824492022", "29246422252469203846996284", "29386096276019991509643500", "49156718482999311956010351", "-146178818417047615416629788" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-113609160992875298395963861", "-24138001744338957650286105", "-67004809197973811498505516", "-102688488214775613024684556", "139561555742232655615687767", "-148612499920539242012129054", "-60197558339677876239964540" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "20195825920279795940729324", "-119334106986421277897505308", "151659610074839765261403354", "-118170878491501590658833395", "50153305614953569413375206", "101283064071515743153346905", "-8052314554905129705402755" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "23932763385345968456601647803", "15652781673781689996402901658", "-25714911140136998783717424112", "-24063108817371419630967954407", "15277383588988737590192970511", "-19221555964336177191153095738", "8525778421472359673700475873" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-29702031637985515415546806692", "-13859102307407213719104673225", "-25067618688733109656139385633", "-24682956579651842668991571498", "13711565425088715334400148186", "-19366222915962629805772891305", "-33545282070076066792477610200" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "1577579865090943191123249075079", "-3525655228029257341271764365663", "-1050223731403232946069011798659", "9184101680926576354398417891029", "1090277651050981596352138640337", "3043069214224348874921088096672", "9123125546671244166682999591141" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "5230742018439185774351510283447", "-3712455999261483372027089691766", "-2990152400744505134513538814664", "7316572061546595364944616807071", "5602827451767519820566081421704", "-9545247122155861772969509882401", "7032145480601368225417113061961" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "2664567996265385909226813270251", "5361353871754409746653525259200", "2256932189357666584876220153595", "-5654300048316572634355784474153", "7631572574661069365537533985027", "-2645945495156645486998250219170", "2020870690070534491429622566455" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "2007985045286249696289152653639706", "255131400056657258321022115794563", "297050787060326049481336858216178", "884106234438982336563352611661346", "-2587734425716673025569123990231730", "1957450916757365456976201890886857", "-368089571172755563470896249505623" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-1196387814490529999805386914015829", "1038685272748955982421814739654364", "-1193187060865049033560022310447241", "-1172542451530951320883605317070612", "2109806848803753145281118730755717", "-1123127061355414245949691913443684", "1576461862603874408384234984377650" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-622450033088710447603286926602835722", "647404094951668977726470009898385198", "-226340752111389342737654608992675255", "288035057938916398743906657372237781", "-398948521522675079718064199905049101", "210834703229901379528655619684415338", "-151997602879056519490824338774903819" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "599661339271767435241765491583593057", "-405698515950459980602780956812973563", "-651291563428939025100760124053290332", "-657745631936196547863976296056517471", "-600407163951273202005007415374308699", "-238929313727252416233275164296771369", "558497244260050726702362474833539378" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "47697045732499570659740940309604340801", "49094216418383588916902774794940184651", "107458416864580553906480322750546108468", "-126667437332361921585431044198585948515", "-53936578834290623326458038655308336078", "104673595473921192102042280196362912884", "87274371731568548795590851613278065574" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-57994691774977410461009466091670483898", "-139290829877786761805444219267269803497", "125557441407240749013229104923768296949", "-149517211669454181272347743110271361442", "-61144120978857663327902638374458613629", "98284527118323795858458311216632999342", "-121317326075664247739969129996213981006" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0 ], "DATA": [ "31724754377471299103514825907465147200", "-68683175193407119606110759218116809794", "51929217413585477513343109403149838340", "-145027261082478191639375237467632529703", "-49770426552521492900748892821647544270", "144672559328394602382003101451778075572", "28345129948566882228213841519097444442" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "19013", "16476", "-20629", "-13807", "-4458", "-14679", "13895", "-6591", "28846", "-910" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "7624", "20461", "18204", "566", "-19039", "17312", "-16104", "-30039", "28542", "-31955" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "630872", "-3149478", "7212148", "-5625858", "4187377", "-7592605", "6361212", "1360513", "5585631", "-1637645" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "1903211", "81573", "7930229", "673455", "-5195870", "5653300", "756828", "2999469", "-5710166", "5301041" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-1853757120", "1047124920", "560213792", "-566455513", "1050311723", "1598909497", "-1861461622", "1007453606", "186779504", "-563460356" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-1383656932", "883468748", "-1292278154", "-903131312", "-1071814745", "-1300060646", "-837242798", "-200746325", "714599424", "-6503541" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-460640621", "1452598777", "-1043355004", "601836614", "-387360778", "-1842549950", "-1286045229", "1011279045", "897922597", "-2075789221" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "268979329564", "516400770279", "-330547198646", "-138555837050", "326543329356", "-135436080610", "-179952442222", "300111077580", "-276023308365", "-379534546872" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-11746421841", "-355409685643", "-176229485899", "-335104431722", "162586810440", "-417505572341", "-327503218560", "-237592706942", "284154570293", "-247114028858" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "81497388294647", "-105089606867026", "-85485884825237", "-84732918845306", "8357901780398", "-59052160981538", "-118765758366224", "76774046345709", "-122272430514436", "89632957325007" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-49023243323234", "-19691482514190", "81020005388878", "-13049232621282", "-64400521716964", "-44556778372970", "-126760338329887", "-95545268831523", "41700405063963", "108419780726149" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-107526557921638", "107593946350906", "15717527533250", "3802176381376", "-95371837173682", "31982300862443", "11262311488441", "-71783765256670", "7543990555883", "6433455104518" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "15370190057594307", "25338659396844360", "-12395415092069244", "-14339801329528901", "-4669889778034838", "-25293753369826946", "-3864029849168179", "-2361320120283656", "-1383816312214096", "34795785958072458" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "3783311029197913", "6715667064352355", "20491322855651616", "-18892806810730476", "-29298088411781325", "-18004762134322096", "-6287459475318319", "-28607277114462589", "-21423194975668894", "-34665915056050672" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-3801841962478573811", "-7405589676368624854", "8312654202555889027", "-8469153922901405899", "9033342105804955460", "353262572711500661", "-3106215324351420666", "831047326133816380", "8150006237115616315", "-4283399264798386963" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-8901584431726569509", "3464311601859524575", "-5869426028134580414", "-2688169706060456011", "2295784898660947844", "8661201927268530958", "-1291556588632435723", "7508890651106821342", "-552620568374393213", "-2414552209269209516" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "1268774911788677821466", "-1468173730086716488054", "2333556357660662925914", "1125086588336622451903", "-1307938906371302432016", "-1335315025974812297181", "-1770591667023996442654", "-1618151743678349641488", "-1953870570892738899528", "-1420698506208780653516" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "97145774678299519394", "1227557973951035235823", "98483114001898133318", "-241607937190185623252", "-1824624845562727979837", "898642316179641750374", "-917227455439067325819", "1391756574048152055440", "-2338483366275639941073", "-236784562298273813783" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "1385658483208773656383", "988352401637819464904", "-834024798417789410134", "253470261013770477298", "1027824244823037033941", "-295271284578270208942", "2092749933699556157881", "-212909696600914004341", "430816063369429652642", "-2118940369361998675417" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-72495282431672509310211", "-260808002784871326009195", "367911180219297507096976", "475567389581679354296412", "283347954325215180669562", "-247672665222729098007990", "212556445759539758862792", "95137611300085883049338", "-253373421791320529250788", "222529485178868323798812" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-198192142423639484697068", "177859535515358311773279", "-558532822597349100035295", "-340243398464679108564091", "265469123993517923432529", "345718000847558318478738", "378018566902132428188504", "105885046930552921000134", "-160893570768315261756622", "-387610702742418384935858" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-144968858380916871672160248", "52324015489202530187648533", "-60539725259061399057418114", "5951149345478800610815784", "57415730864791909115606042", "-52801875272942350510376204", "134592231723492901663336639", "-140309028873563970501229424", "-108507437605812788817593167", "34131217432985750294482026" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "101925561886985522713167496", "64068475454853961876718594", "52868179913899871904937921", "47228119380503897884367815", "-46788993674462462832287570", "-104610259825858116764781983", "32354527810168930868473679", "45056639449462594887058208", "-135796831562998221931983419", "-118863104811282298311979720" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-144935023148179749058126736", "48227314826112309666538622", "82605524173281841359546240", "138426913971094444509629819", "133303171716402294169615293", "-142698067685030493395426562", "-16118795484757783951829634", "-127345535900813079037928402", "31052925027573527366876496", "6085416657630733491512226" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-35364635610587093818111370252", "4086803468490861639250374276", "-1753919885518013415897000208", "26862553012392198245742349664", "10669372287960510828219956384", "-1007097557406650376921479141", "-1580693773196331906210657222", "25812917857711965549650468024", "18724993038796357154863599325", "-24508123889671650819840888793" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-18902279341581100470708119631", "-36522283854701472155628358904", "39532617789123607497080494567", "27117584983573687565416505488", "21636499255472286057566042509", "18733621370093464247720680288", "-39480534968471587456182921828", "-37721391776569371748588571168", "35468843832781032242301509843", "-16165717202124486435967229402" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-4522816763549834982990574959364", "1373820747630420236782319267397", "-5973300398871900188157345915779", "-3481250147289716375798626146235", "-1694496501940066046436069391978", "-7462383927751567171069774651591", "8912249954123145763780574884843", "9381633652197120079484670939176", "-1098878014354252629151676845724", "-2173222031454195847214087975766" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "8975610890580938533873895882166", "6181193231273971804885197385850", "441052995054523394237285556116", "-8766763319719487961533692298280", "-842714990971736191972834107071", "6840968783732306639312843567376", "3841095572745602224231284364820", "9467751369945105582701807843797", "-5351035191765711075838610630694", "-5308772832124983511335493031630" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "9347392750318791423094898319330", "4183549304829613232393877274325", "-2300378611286319596549279057011", "-6937551197751554358853541320617", "4564202808321596383974663617261", "-5190481232725371819825118580382", "1822212646901438322631749853066", "1485920178528983047791595729329", "-1229068069521960212083202938332", "237689578861440280652794100924" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "115106178081535021784505745982211", "2419329305084042487339107025104645", "-912291369331954853157175763841502", "-900563205408184843681209632304310", "-2409941993002043200510899606638580", "207954557233582697210944508650947", "2523362759243854886179529293350534", "-1209388681794143621587165255741373", "259026751367118542127035660878113", "1003660757951177805123793137984087" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "1364136601439660016967396025012011", "-2319775878890049929548330062341070", "-1449285195280197573540980195382578", "107888626176858249017005749097741", "-2047490557041291301727159113491968", "-879558452744622993459196325149602", "-1386527790084045558776796472278147", "133271536923889936499750446388741", "-2505010338752155970620613001087615", "666690558706649829294657481518697" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "647827261339309139672072271604218401", "-258811588602379592317988185007366025", "19118764232643136124664526632131448", "-342358790376296733951169651683991194", "323251023333708738421455258993841741", "379547465547461884608218077364934561", "-377241696764273343171368191818828042", "-474120316689176518053512200120507400", "572902079758457973787032998836436243", "-170695774138811733345401262184630875" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "305708404293242487627320811047392371", "-531504659332015189310098462346024467", "38731992465228164632556252163535297", "663006990236245838431214438209595319", "-118885733274822068648133582054828225", "-144829449643189775388924778145388908", "472872218845792265593284840437270542", "-310671243825518439944530367781619370", "100999780786464624037618413063970122", "-140342096347493988777682079106141362" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "32980532793889149400753838532332705439", "-164344442015956226607797018737685311450", "-141620964137040177729886345057833620325", "129406137990155072039956432976677981821", "110860822470333733093650034824108278208", "-55257524399499752297571098737083027353", "21667380566056781477986443622263599784", "-139527484497317282946286760556136131579", "2993193228803848771670718010890478039", "-17314058829458659076458760524200313761" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "136785414456456523483802852002988078554", "158300488297788966764944523807878924712", "-139182135932402606222753976139330995413", "-65945426315720446378481044342262546327", "150507222652197288662681358385272850559", "103871248288732570387585721016195272008", "71995969230355239685697947093141718491", "123362031595790252645978327397128737346", "-9750682546488018096795492364860912234", "-154627508798373930419429872814370749557" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "119842907203642501605603704672016604700", "-17268268161596497482469861499561728202", "10844152782523585961703939469633384076", "-62997709460500166787346095846904613371", "88308111493725127276829193825175861897", "51565439967137952485926321683961024024", "47601722115696675135575316380394827453", "-103072164908596700312680385416990765141", "99066862329124132138915843909816085618", "29881936603443354551348559988907820317" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "11695", "-23366", "12098", "-5922", "-24360", "-26490", "-8451" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ "8356", "15273", "13350", "13154", "-24436", "17249", "18193" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 0 ], "DATA": [ "-7959866", "-4584859", "-1849516", "-5743547", "-6549993", "-2265282", "-7704024" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "7862991", "1982106", "3725606", "791227", "4292112", "-7820825", "-463905" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "393711805", "722994374", "-1096064610", "529484077", "1342578673", "866470525", "205716816" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-791379162", "-1640580986", "762691364", "1995469295", "-421319712", "1271534201", "1429378995" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "302750528", "-120846658", "796589794", "959422940", "1637146863", "-2105168751", "959083607" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "369772287616", "151347559746", "310899836472", "-72176434390", "-96924947011", "-228897695327", "-200002009593" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-457074057554", "-375995209273", "316810959776", "232806094204", "494582201448", "527142467595", "429029914811" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-2749358243231", "-16732337474393", "-41981779092256", "-31047429629139", "85922781452400", "-110248875388850", "14775617632206" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-89773154375777", "108323854266884", "-25662671385030", "43022064801421", "136623279644254", "59073010679225", "-68467564739828" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-72813466497130", "71140769259291", "-80317050256498", "-54904445398065", "58067517976047", "140251327559406", "-19836322904046" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 1 ], "DATA": [ "32549494969775706", "-440355605481950", "11791444108889307", "12475855272248987", "-2784519143284935", "32466210616699342", "27675300306271448" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-2902857186001800", "23303428815434999", "22978749219565527", "18265376949989171", "31457574635371153", "-7781184835189876", "-14565659292923040" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-430513575868348426", "-8790392468356940146", "-4699312496566636040", "-3993022130065929131", "-4879440303771900917", "-6552655591468880275", "1746806308890936437" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-721862386033128083", "-4475260144423557773", "-2570981490582934517", "-7711961915298512607", "-4822749646811070045", "5836898402712974203", "1646816636258502903" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-196690994284317985909", "-1104147372526288843337", "-1412688911560211238060", "2156517522260848540623", "1969864666336817403192", "-424864910108053276903", "344362455866908271873" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "1977829057280933277101", "-1853366049465512956083", "2335230945236060536049", "-1075731044607594025734", "1859892436977688408623", "-315861879615022790690", "-406105163821827431877" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "417852109818843056255", "1752679773930708218188", "-1943388216652054599366", "-714786275398685414265", "585659101400025508462", "-2096928276954243783388", "-5439936016422902751" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-371691511788058976981831", "468331324108156702082525", "-306216617268143895601700", "585404166947186691975366", "288996272789179095805972", "404343176683141658586544", "191866472956788824823964" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "315327112788086641525519", "196752770980596165008391", "-572486369362609381413082", "157070379247532783973389", "526043469975661293749107", "-332418593565470406233968", "-456053081488540795834833" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-32587082314199631587299920", "133861603238939464220682787", "-83661874551945050110053855", "88800710199138738791487458", "-148303404783980031631890281", "-92723105772899258187193672", "8503332868372431603113821" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-19332793743633931994349116", "-96326365761369957850035834", "-131011081713605299896434887", "4272403363996546916496923", "-136751423492618686805500248", "151616186823003014418871731", "-1671600429980516271659907" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-140178641895610194038703824", "-101032891978279919640123306", "81660784678037787563074285", "39964525581604073164209878", "-27374188725798306962666133", "30132624370004367295733964", "-24808854061327935626508114" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "-18346621296739073329815384276", "8129555162832277260205547644", "36653480194016965780646040547", "30413178297188904152302651584", "-3894577824082035416332316895", "-36007834468089853892594019849", "-15800624696484429406311297074" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "22023236505727198042054799637", "25162398511869781136612460948", "-27627237437030757915583163269", "2399929206249180782631036076", "-28662435722237160171923737772", "35814219127521949420744820118", "-19892289081278817747121556433" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-5833740800009897266898626041467", "-33244429845422337192185559070", "8607576650630464089887525333430", "-2116847884082967129690884147276", "-1688186241717588214733687423048", "8336863793337204216701546448411", "-7776559106309167653701679757019" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1 ], "DATA": [ "7677666198324057071013895406917", "8312123351615753410594523358329", "-9172950392603423861689575769225", "3983044414175939736030503134666", "8631672985863441241981992212823", "-5092486822105432034536308372124", "-9513508243773819164622502221862" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "5586571833059467170325581422044", "580606001351331867421453366092", "5459008382235569678361699614142", "5167556446031629490226831649463", "5538049048887584844681848848766", "6285520963756397613259958855517", "-6303661112625789803890168279755" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "2026431296682674250857553865416840", "2156728720055476065414283902556465", "-2113626044266072567032924427618725", "1577105749037891124349723340806308", "2341510691513285685324249715992360", "1152985290466370765853992549550372", "2257606784668378735530248971238219" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-1909890587318622197067905430014808", "287541947253583018938971686094227", "2133113520020164331925609988632397", "877412966607662750851739644288192", "-1354720000623937488986892371455979", "26739837041119641398066396760126", "-188838172981149861365490159579856" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-474782395129122353298074619574123333", "475246432403218151819336842343219369", "-173617778400150860563781368814715566", "-185846112444193317541092054250459789", "530657363533857853032769680847569540", "-624548224136809783035837658411029096", "16767939677460215918277184102392562" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-8794439856560424693485039347212630", "-74573922642298735959395658944476425", "50121637564100226023942304740008235", "100186315919309941191487123083722961", "-656685831019371144979565967330203175", "598127327210536287666688394187681062", "469011031931868490962046889570742506" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 0 ], "DATA": [ "132571096983923187471923107191482189670", "159416816456247505951844178439417096312", "-46858620574250494092983344457915710534", "-32188937023483923298551233297633219014", "23399433488123673778344704553087116420", "49411831195906345612909906764377215656", "-120433578968253846639971258003920226422" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-79060725002651426090157731360159425117", "23590752860694088349229816117245792147", "122068332660626412558932914786262751393", "58567440788739372959622438841791498887", "-151757958624575453818610406998254514825", "-46670330230498566773973502369131285704", "-67623110355947762123153960505167699960" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "19323325648279215563974499442806584258", "160743730096300729908285974982614804782", "98979412153227101016992388437605176538", "169051965254068352712380666347973917472", "33410045474491953996554951161727411456", "87639385406550804599722815989154690961", "-127263625338574842387718381281492249303" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "30246", "4750", "-19198", "21754", "25255", "-32398", "-217", "32432", "7160", "2502" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-22430", "3984", "2068", "-17474", "-3604", "15015", "-28407", "13805", "3348", "31779" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1, 0, 0, 0 ], "DATA": [ "2080492", "-5892388", "-4427617", "-1926570", "7753036", "8027624", "-2138635", "-7355969", "-5207656", "565924" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-7581394", "-1145315", "-5530746", "-1343136", "-1965882", "3388545", "5063475", "-4724789", "-5369747", "-4555562" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "12542417", "1259749219", "-475800794", "2133922027", "584606746", "1701844663", "1570293589", "93985663", "-1590446304", "38383855" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "1335818368", "1814707665", "-1447700920", "-13690005", "863389254", "-1088930392", "2068779612", "-431001990", "-133595395", "-687033383" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "1433448420", "-2037717833", "-681228088", "566270991", "1714830138", "1383333290", "-870073230", "1018390804", "-1658508550", "-1107870810" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-183566186766", "22430286822", "-216962582571", "-466722944551", "-27556463961", "546569939868", "-276694721505", "-48879453774", "260500893747", "-442983780322" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-537420559602", "-20381317872", "-238654812999", "162311255965", "-380528677930", "460248798746", "-470403136829", "41925493634", "118729323873", "152145641309" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "19115429868946", "13475764429133", "-134296438665814", "106743314332396", "-53710936617611", "55826528957975", "-6112400391073", "29771890922370", "12948136568295", "70280112187588" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "75635401406293", "29801992058998", "-104075542445723", "96149637189488", "-92247279012539", "-15060636948068", "-56990909320532", "-42148465517830", "4636483585350", "36352680284144" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "109325942422503", "89437875709036", "-69763342392522", "-109570939746784", "-51557696910336", "106618186702949", "-52418370534901", "-41131112323835", "97009168063041", "15052093903552" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-12596475918000306", "-10440190089661389", "15930071291826631", "-14154914571137080", "-20679980275492503", "-5555224487780486", "19901627911606993", "2179801741655552", "32519637793270999", "-12190525011728478" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-21675467468906896", "14317453815507613", "-19766776814356123", "-15076731200801441", "-22431380700345243", "-6473570686950262", "-8373384674763888", "-20901722770742947", "5175325370040911", "24214606141441984" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-6586995582558976488", "7822902623338681773", "6785324312631495786", "533690903523561420", "-6270244316286526849", "171361209170531205", "-8970550092894953161", "5694205347266685846", "5128833225945547275", "-6526220735860764396" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "1525290815331568444", "8786542072042599843", "3211936611572870417", "-1018796928748300854", "-5419478240797376370", "2820144392948109276", "-8097542825869754965", "-1815427842766140431", "7519915858501526150", "6718055695379996173" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0, 0, 1, 1 ], "DATA": [ "1633852390483743207506", "-1111545040340809206837", "244724811785233487882", "-693891632794029583107", "2048439746612005438484", "-2139374266382006896259", "824431366346499761550", "-2178751263030852933012", "-1927196521844590849943", "988141071239138916737" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "2257309169850384513902", "-574793864866852081089", "-1948188754190708684653", "-1683360294804356257358", "1047989496843790848714", "-1920909202759912391700", "-524370318736298792785", "-2270280452792278665082", "-1977443531683898225523", "996281040520873028206" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-1794161457257457767507", "-579049890249855445335", "-1877229817480332151412", "1516286188346047750655", "1679089559862183244805", "1899758075679851870084", "187132781949105016179", "1867935632353572731320", "-1296841752077650621812", "1300114283317135818081" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "158042936756450218355580", "490847485432966130350847", "529866300289645094459968", "-197353870044211794005526", "87425841556036243951489", "373006885981853652482697", "-287764822688722336029390", "-190352821874203518937063", "279464221696482690177922", "456648278931495677210040" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "62449586174442498659851", "91984117770625787970323", "-386017533006981165958695", "-252962587854796012161774", "-41590821064900391076386", "-482091979386779184677156", "91503772892342811308019", "-283504385220883938631335", "68755932698885547223664", "-379651999761873616235097" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-126087576931161225003970774", "127239018834369984513800503", "70141064110265604854079590", "154422224583900250448650270", "77231480791207775951166590", "99492320089316975380000682", "-153576307293383539264890034", "-116341636204244696735578596", "-15786876758860723008109946", "-34149255348815753839077980" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "143153832119590577378917526", "-23514921785325046830312515", "-77172712985345097430419628", "-14243153727807957632994400", "-58696496713845538380317529", "69538635574941878208722809", "-26166185622705445056596537", "-62690164472796028840316596", "21101480017403262376196510", "-26276281990810217165918990" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-40333807166719138901605362", "-37987667593309107331951459", "113271936526201411790724188", "-45961181592872567354204492", "-55914627445218102108646757", "-104465866552974080878275370", "123873735196193599773252021", "-69984072371083501932162355", "60408753566988145951843027", "84960611400280128174752202" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0, 1, 0, 1 ], "DATA": [ "36246979996368740974728708372", "17919686541827178802317575872", "26685274428819984663215559869", "-26830140388908281565789341267", "-2931873170900073034034488176", "33796846013743474981760651067", "-28384722395594446851545817635", "-22654753522493569083372946875", "-32663458189555845828697337353", "-35127901885613742916339593277" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "20660444307893572140115730346", "-3785075277916670195560917943", "505119675982960330877221727", "6140046975369405521925948823", "4736562894917206280220185620", "34734639626599117839339792477", "-37238839274684550289419840966", "-21192307666520898522344443325", "-25731253359857977263230009203", "9784873768788010897306473642" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0, 0, 0, 0 ], "DATA": [ "-5228112207960439532634245527389", "5167964205023544904856267703266", "-3153830415354733573037966725748", "7450883686345014593203587360908", "-6190984559160100747667669453446", "9158379943043982798000435871289", "-5143633836669582578660334205312", "3857866493487182226354931788449", "-1995529191506094552567237788773", "-1939757494685073234017884338772" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "2983226908002458413591863312573", "-532752777081931294958349500501", "1691976820460575299172383596135", "9026960176016243081320400536118", "-6149275498377603931086308362289", "1877762879546909895671695219561", "-5543502816818045577508158461912", "-9721729867149888495285508938337", "-8920180100768743706171922537830", "-5169979352443901899206068056740" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "8702135094804887254191507204010", "7975177441141361458695114972857", "4215501985734737238170439787178", "6544777372478132898809386710105", "2765576461694941619080416997749", "-1508947428389258215537026450679", "5110812407105601388124457686907", "75095474631054730513072856363", "-4279937210566724408068745471134", "-4330630809933126585531848495211" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-1504559618154883384814296581669162", "-773390196196490039153882388890703", "-1897752906170923065951917060777094", "142958130493415644926376088950273", "-779836256990434274653725306832085", "-1753325836605413516914447603525700", "-1542378627150307800859873587627455", "-1939970582128933270354383457813584", "-1411234208839859424724556632640107", "-2126244899381031545661748447926471" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-1307160371865080630115651820675170", "456003844170818774621646351482357", "631068765196513436260956185922434", "-1965243043509431543877873260642837", "1924094736312307345032078234417418", "-1875239514046642535329927412930852", "928252274243472509987779134671194", "978995848734671073561475467379020", "-2564475835488959647742888457890405", "-570152793522093518749573568631120" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "480389537893603636983558781276270114", "-322627233363211525549283706077204323", "-175773029789054851482330028411371014", "244330382338385690861905900080229456", "-392607940616669928826392153052775593", "-107267728131376637957275222403768541", "-545143790608966683829710014599747249", "364869469591341322932716899752758323", "394858084329837561588098411142592437", "-610840159277877166690985420617619153" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-192669740281245461265542023292550989", "-12603685317720603777562699557006198", "187283379151228677500952715809692379", "513546955692369120286734180331431234", "-482110749762971834845638869557855315", "91471688959029993916111990585487772", "220809860807910859636036119386009639", "652850877928760022088469019476234455", "111668879690942453024321503787659219", "73373210253636230671660785291826427" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0, 0, 0, 0 ], "DATA": [ "44996590607819977856286386900323601897", "-68502769785321582579070708156248378078", "114506708080602131052528148832194410156", "-83232569015900232819092415317550976561", "-97716408046524726095696560404660126718", "32040614956823782468855275605877008019", "54753377444348733160915719409913741068", "146560799167787084071378433728834118133", "-46659169011613354044672675978630700660", "-18373617990019061846229769982316099682" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "67505677281734524965060455064837775711", "-28713223195193703313300117213126061466", "114670528108596039585803474305454053759", "-109328515397473152172020766238706523521", "-57747808164476452164582125724287659511", "-115039911119596976728830311034338189054", "90118785065736856720547824894744092178", "-89659383715739235835540012679951558145", "35536568601909522172926452268384931891", "-133420441752119085196081353415527833003" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-8627303427894096841021728602542914202", "-157430702291942847264537187308979825421", "117495445521490567278694563084720233793", "60657311361342276211434917495326305014", "-46555659115647145900825174050414002012", "38569430641717011417297628092264146455", "104117949912584083295919883245212540224", "126517129168323484921506157524766805721", "-89395247495161073292219697890210210916", "93186204422346566052866526301928198107" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-28716", "-7461", "5893", "1161", "-17035", "-15574", "4237" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-5295", "-19936", "26923", "2510", "31124", "9353", "-28774" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-3553533", "7148395", "3061750", "-1009165", "4381645", "-1818667", "-5730247" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-1477251", "-5800734", "4881425", "-7296550", "-1526219", "659450", "-8356198" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-263873982", "-1472352429", "787206308", "761788112", "-2134545176", "1917444990", "-145750272" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-433966181", "-1545963916", "814812077", "1593862724", "-701833517", "706907239", "-544182188" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "805594080", "-1195679709", "882347557", "-1853839955", "646020133", "1484971204", "1300988856" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "4438499204", "135820810048", "-298194796045", "-167913299917", "-257288837544", "289738017191", "-418170170454" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "17906290840", "125414876927", "-275575916904", "86104924740", "-164076860424", "390339975362", "-408791420440" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-43710596027703", "112836996919727", "-105977145834636", "-112904124134240", "94001656040741", "65183044734455", "-131866661735203" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-90977051496274", "-10596476734437", "61560055683609", "133871189520814", "574814014019", "-38018623146961", "68859957958301" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "26944281635514", "79179252721889", "120097494144511", "-76597855106777", "72037045699022", "127107312225912", "-51976566719455" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "386005180963195", "17673684201876788", "-32819779628129716", "-26142734584760798", "-21415081378865369", "27386071060762895", "-25035767854560405" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "28727678942578358", "3716854509922265", "-30765891503268996", "-35428939049280802", "31032813396558579", "21132833493728424", "18323423600720926" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-7388758649832651560", "4890796540741969531", "-6494554253322577454", "5501693823164720471", "-8097047242278575855", "-6038810107886633396", "-2806525281648999824" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 0 ], "DATA": [ "5813079697128581888", "4678184506304863002", "-456855996733358303", "-2121277677973694726", "-2248413943539318365", "6272381020409312259", "1274893351361543565" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "1566501292452282639110", "847789298808224186659", "-1381494557864830190709", "-389660191209952286951", "-191009516626814231082", "-96116899012721630801", "1868221459505859782879" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1469408605037106208705", "120169104670163997156", "885213643526004078499", "1656898855207585194216", "1868990072585461344802", "-933261783269672604472", "274307016875459849352" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-1897801203902311359499", "-1638391687561633219399", "809233679262405093278", "-1426552107698825838210", "-2156155154558519949123", "375648567641874627428", "-83641445825921590041" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-54554428886951413193957", "-422146139334599760408373", "-181537137117950003153154", "565479726514324786737359", "424173391709408884362222", "-426382471585898970959321", "378989549983955912678977" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "332482827287198457983040", "26269806073413044497682", "221494228699356468610270", "-214742700742291383196050", "-485063025095965310275166", "-206922662880263609184941", "-12269499779054433613668" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-2079153784240544880908553", "-108538000488167946369982799", "1337702113416699090369723", "51201975526129906812967855", "95133477031983503992368010", "99560753378096547573231892", "94680749872380168429064405" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-117274302077491880342412021", "133235309557643707264847345", "136425992851983814996249850", "141388968148866833783730856", "136798627735587050485647952", "-19126693188647933554261649", "48757965489875879851975965" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "-8286635741174299975868485", "51609959172902905910311173", "-109111488591575501922430237", "-78557116791558802282576538", "-80656136078314199047657721", "-113053382890647628720214794", "-118160471943246572510407260" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-25813553783572142627468328674", "-25977427310356228285126178554", "-23185854957388926526847402942", "-21707862041932744516393466772", "-6421519562360989032505224517", "569338598241172908008799687", "-31836549906764569188743203758" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "37793044332277423478767661867", "20748364196925817852337750787", "392549789846734208161909185", "26297790988587948005258680240", "21586249354499693315147989389", "22288577155677496694419036361", "7345604052580273885509583438" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "9904495274881627900139080199187", "7482827727017947275973272651641", "5495560179907337187222597837509", "-4055007536622081915505189176936", "-9465371767163898005811811099232", "-7331632251430461716607678930512", "7141099397756868279221372659248" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "4169713351778166668753234914191", "5859067319849858664254420101042", "-3754282354044818620313472353105", "2568048327396196032282025142854", "9311607892968389058421897582763", "5977797766778831148484981898236", "-6645965265653096929117017458058" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-5801906969242261868600254035787", "-3336257298871507164494778520275", "-2636835112016446965166101852431", "-6885572586792737606252971841764", "1603907679825345662447639552058", "-6879056187486539384923466888462", "-1790707021629779711475082448245" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-1034675525852537762364236474862575", "1711882430155290136112735898040514", "-95920345818202387596164300835452", "-2249141525032950659874767503830521", "-1652263065729457379526511839069047", "69137670296199427018181414584483", "-1688424122016258582447918142136846" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1068287357075711311089614491385749", "-950554951242207664534573691846304", "-1956036184083258810589554917733280", "-896537215061273632781289017902857", "-2536258376526122583868387765640523", "-1792994625229025457039629491080247", "1098569487026708584137620128184550" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 0 ], "DATA": [ "175091933681780709419449962505368809", "452587234945438576574474457110999168", "551706621587607711559343522731729738", "205869761865384575133175619802827356", "421828722673799022708318616794744473", "284099079750264743351804395834429375", "-227206504976250724681610321278346086" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "653258876517803228567267877074483421", "88498832885550519275280233629225185", "-76011122884235139182377528836347729", "-306036684553026363758974895394410014", "507096898707325051267095986297417498", "7244969811766625291666288991989734", "-573031189194397866985879798798669571" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "-78707895658241450417047565739309187422", "-109360485739432018575519398965631343423", "-2428635583984270308797486033895962820", "42753163486976599252181370759774389112", "50819284555128794452586051816572760071", "-144242532387316071424543222681940814787", "-36156399915684139601256093989051959166" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "162984880645317865260475927948552198958", "85240365845562965434048060592369612866", "-158859671947676245091304105755240830347", "-106305457729982387569448077637482766080", "112063647808441935089339730874648100522", "-89567526888097522782942387372735070337", "-106346834442166901546233152794820385747" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "49579890402336664433922726332197359945", "73089275354843686046803386174212568686", "-114653905042244944982579349957812151500", "127934754145313083729997784370486712512", "-153960964783146902490613407319798440760", "61282287594543529304868203724834609850", "-95735253186395895118322435677049180510" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "27248", "21926", "22498", "-21109", "-13865", "12084", "-10974", "-7421", "11109", "9914" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "24181", "-10020", "-21281", "21049", "19869", "30238", "16713", "3878", "-3800", "12113" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-2112580", "-5802445", "-78712", "-4496866", "-5053677", "2079868", "6150553", "-4483834", "-8023488", "6603846" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "1291524", "3929930", "-7350947", "1175586", "-3405008", "-5060125", "-8206503", "-1303953", "-6095530", "-2760640" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-2147181345", "-267715231", "1287957097", "604610482", "-1548079692", "770882347", "-1892574560", "-253216281", "-399486830", "1325921155" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "1675268343", "554051112", "-1764481009", "-1789514237", "-175051236", "805874123", "1858433827", "-799955854", "-253242360", "31754547" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "861980915", "-1734894499", "5375232", "-1346978107", "-705850170", "2133236007", "759674577", "1654618415", "-1064241799", "205825786" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-439739629851", "546687129596", "-183193050720", "-19380489102", "-295956872287", "185301861617", "-539483275533", "174936140900", "-216157665368", "-323903818433" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ "-206902602028", "-25523711337", "329376290572", "-547086561793", "-420540489807", "533906771992", "-69356183432", "-15636527669", "-53282559214", "-290867614525" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-71228950033910", "-25738844726735", "7537765437683", "112839875804433", "97538539157933", "23979706973815", "99830033351181", "-98591255279452", "103618021816339", "18626785293021" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-63486282809713", "101701065580704", "70027248433374", "-55296910174840", "-79310707567811", "71500005915799", "-70746573691925", "88747230972798", "37301558421391", "51854510134738" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-87693600270321", "-12454309380745", "-78257696973308", "80285965677822", "125254246716243", "-117675991977555", "10663316581359", "24255903525594", "81254209111391", "-72113987036299" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "29148599662921759", "19685775734681446", "3333245217877997", "2995823413324396", "5176366661056475", "12337102603147756", "-20628365543902019", "-22418243161590754", "-29067802239769240", "-23018730875097178" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-6176426923194882", "-8149067582880393", "-11749166095143035", "18401921003901093", "-1996265112064578", "10197501871192290", "-32253621163897299", "-20847987719333936", "-10659701083886535", "-8278323833998893" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "5136857587879831048", "-4226582843485454109", "-4390294981117989663", "4994818351801512739", "-8267282930090047916", "3039283087909496060", "-4294189184656602394", "-1041549980371122641", "-3656126503669238317", "9105664583047399153" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "8890565452093789405", "516172262246414024", "1562263263000381143", "5536662842752237787", "-8351947536553705606", "6464103264200892007", "-8790661469847258111", "-6209442610558714656", "435574662691177275", "-4406978950610351547" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-288778447471678108496", "217217798831157629251", "-608988687910814932673", "-2240048037329221421107", "141299084981472691413", "-2195387109122670914754", "192149022897618665404", "667039189961523482717", "-2155889747007763781525", "882853814250996329398" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "1751402159298369348560", "1451882575998488934096", "1558108778681655864225", "820503763199449595318", "1623615880148157710724", "-1033406684162170713762", "1930159737768844808076", "-940015457688702701592", "25356638204045989132", "758908114106815880742" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "1716471103883506022556", "1848997534114484794729", "-1094717836300829658295", "1157808654390779724644", "1974960526678787421302", "-791852644564930461068", "1518936215943205538052", "993164305481300592037", "-2190917609278580708946", "1868797478910203743832" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "107250259596579898525274", "36231469749575876646505", "329436325938797206808627", "-280578826478526573569133", "522936041458815155368518", "192285773258090139021578", "-54285825226099356185035", "-147961354881812346695825", "600347692235230272507966", "-353688820541444702551558" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "365491534656040934517771", "-547465713497612713037502", "116041079260709253587380", "119630447821474156181639", "522081240494430350219263", "-474030018965150641292577", "50270333922267433091553", "-24643818854330102652483", "245112928498664779365511", "-121521490474436234210533" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "26593659860935441804716440", "75646833540201692356918621", "142130608477738687479162886", "-83412542972966251252815465", "-151155951774532976658715599", "76154233043964388249198213", "-46853272352917278017975341", "32486164613383508372076408", "-66829799824523238584057062", "95792417909842220506235101" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "63730088675558873019939745", "143548109718428079482639663", "-27693260291056796500091225", "-72348766321107397898380234", "138677152483427690881537874", "58516156596033347413669914", "-107433759528371577596908748", "-78141621760355595625354145", "57091054796595629043991192", "-123271655549899762394142143" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "-38477417050477847045725516", "-95888838803508348417829785", "-93625160413861971562012467", "117087970522726692597218305", "5900192167374488789348559", "-24674580893344344193379627", "140998648377023443587850974", "-66274038957839610112342957", "114522458983086926572397822", "101447863265640837765853036" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "-55906306943028913908051930", "-26457941841314857799301954576", "21759711912953355183243610182", "19401890234998887013156330687", "-15933074126275848181591729540", "27293805787497685180871118517", "13704320194409856542280625225", "25216830024686698097083609599", "-33566241896826513638448226962", "-6406030957083785743431792377" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "13296281254042409316527886192", "5338707057590130523225415567", "8819455897090902612324273291", "10450242868251146464691055988", "3984408170044123068649964640", "31173755255592998054061179789", "-19050412661501894628584863098", "24644713794131254858462909618", "25395276076315730035919041533", "-10189071623516303081731732147" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-2273158346503188393179401750902", "568616855647559589863643266508", "-4280161068916792824269044568609", "-5682425054604041528765672948927", "2331397238480003292876981020193", "3147988283813769672815110099822", "-7632390386845877850077854509431", "9840981189438011830733564669593", "5308479141310485223873660191378", "118932930720936450551429018688" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0, 1, 0, 1 ], "DATA": [ "741957915704735308305168907322", "-9762021206507044939694536775050", "-8250703757145415326282325826063", "-1178202689406322490432919428434", "589668456154125344611661965499", "3829126770246418858041384052999", "-4152282954150010503796558904446", "2039899673016755337154781922078", "4170256258494884707646640540653", "8171545852305751294990084682300" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-6962616269350967896600526637270", "-5635024743757428123953554602073", "-3202590711760239385953136717255", "9910046190612877780929595691738", "-3777668629625810788137576085843", "4614409143748488045192848474816", "3938785692531490617884534958248", "5437946965836538604426507932284", "-2545690782086924734038765495913", "616702678702978670407261429789" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ "-1801934502106174445176675096416695", "-1684879803716391669315795499837023", "1886852307739025680614537841032728", "-2087562101318915052443253201738269", "-289564041897678470358656657180353", "1866276858800525721011618985335357", "1482056324539718482806113984364738", "1379406656844501099108553390445618", "447265769955522315901489749442865", "-1608978736986184364139419097387143" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "959483969247063574072923758413354", "451668303782354619245629961105613", "377534722039293380340942675240743", "122765139177423843232789966201962", "-1265814388489880705763330194364716", "-2555618459717759708270107188671150", "2134470347388221582022629725907413", "104244807033941042242397388741541", "-1292911051966102235019381832015662", "2097241241602887003970802334050941" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "547484906495898917724309897034434982", "187296642318494648639964887497107585", "195010449729172050428280546508810633", "594640376620694404142214250754771517", "616088304853668191153025331194418912", "-379044114359687531180765592396048067", "-486181591192683117321273734043119484", "-334064376837986313664550095407721034", "290486366889561681226507320645315096", "-347474821649864554819987979701785016" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "592690007814067700303615752186669589", "-33347738259818305399177703203880591", "-382688857036730057091326853248715857", "340921527686390758273070134422955102", "-246344114285772780156570736620316760", "-87281520666834090344267836809132652", "537245232536815311537081321427376564", "-493728711296235677585761940344917200", "-377081442166516277552025620543264222", "2719688994411558148859579112050541" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-164648696059184796053055559504365982538", "161879775934873933398660278767200250776", "-29323675327274284512731239968352351966", "22303285730412507017282414138333454321", "37099625903665917412076794028736570815", "-19515107152254927806948029059117184611", "156049973457253022366642762662774258557", "-77241044079010949387624313535905707837", "162149982017470420229015929290220014122", "-165344491933828504403531276296434192122" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "65054581186254715937680104906134532841", "-60909757301953794888323625879233446757", "114089158661229718075421585158899052989", "107965178723789597153491785237079504326", "52247619354554812540456199083118815935", "146474344956199824278086927555782062440", "-39425662132630652063257104658393984944", "93755642240344437526921615504564431807", "-151156356500508700657655200155812034666", "-148636865825500491359194083438394642725" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-130344022211343875529807825195567803183", "40014360360807493456218651568578676997", "135874153827020334699855960081428792617", "120470160757792083712774060829445555003", "-146085386552734357896040388977994934530", "64861335611000408584376324518321919018", "6601563302721623021480813949509601323", "-114901836202314275670611766582662166373", "-106800567037776957620875222943893747296", "-163938636407835983157260355753705412692" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "5622", "-4826", "-23503", "-23236", "-24902", "-27443", "-2505" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-18970", "2792", "9559", "29810", "20493", "-9442", "-2618" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-7747776", "7378127", "5914824", "-7141645", "-462534", "7839055", "-7683709" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "89932", "427016", "6342615", "2937627", "1091159", "216727", "-6413737" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-1508343646", "-714398970", "686052863", "2122626443", "-2055953734", "-1487184058", "-327233497" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "46693013", "-185591736", "-1319806396", "1714735002", "-2133394566", "-125069560", "1637341168" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-24918835", "413164011", "-2057922849", "2056580479", "-457426615", "536434479", "-326452423" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "324353482493", "-224184500030", "-290036567736", "-161876501106", "275173073296", "-357018570713", "-297896568716" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ "-427281617130", "-497367589935", "73820145481", "-254972813415", "-60399473453", "232313028413", "-327759701859" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-54535480365073", "61199946515142", "-9892000149714", "97913524605648", "-85221226793073", "-12545722671788", "-68773831012552" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0 ], "DATA": [ "-126861055347150", "57631355795057", "9186109685969", "-63002972283329", "-58346149025908", "115782687691468", "139849718208364" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-93813523241112", "119065079857991", "-51189229753892", "15646183077138", "59056284602860", "-92669522393207", "2260801424852" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "1036615300047300", "-28482170773083910", "2678469281283015", "25704309658360513", "-12028318394798682", "-17245374013553022", "-16000168049485216" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-33753168399749839", "-27740542059245417", "-8681968124590170", "32278807399459945", "4851945585153961", "14217157851976499", "-8425777401033048" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-2766941444469888895", "1729136918975836660", "-1477513497854998037", "1433050305337001645", "-8233598361619818600", "996969961284840283", "6329284992445382418" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-718575473243337232", "-7375018409999376865", "8283859855328355879", "-4138525682818595012", "-4431847243389561533", "-8084764830291418974", "-3798062670523971527" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-565050117259497763461", "-281556879669412834613", "1705137738829453364168", "-501989730825366398918", "-1424226998969964624602", "-2198902125834852658804", "382157856805932111771" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0 ], "DATA": [ "-303105514873146686437", "-1865673314410343902064", "-1739893893314072462832", "199753736014467107913", "277323073744563230355", "-1117856351014668186043", "2336687337270391579606" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-285211332299958353660", "-1511512024403761316173", "-1568309493555178992532", "1640762644342585812496", "-393568821945143862673", "1038579276245861014094", "-862919179403309677858" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "592348192569902724704840", "-26223245863895268294610", "-338508635422316199664947", "-486334758177154752056666", "-326020328807272087893596", "306693173070483770415643", "-231885616351380144700249" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "147342012649640932496745", "-365590949715364851039873", "-238949724961989355524360", "-20804180906842992000207", "98422693216257524451712", "99343894264393231770703", "5240277192766874192955" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "97249345115448597432142646", "8875195937074039828120226", "1542565882778023075834635", "11772316605485844570699635", "70103217733879642573345968", "-102512631348831506793442303", "-30574606355593089220215659" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "119081714072400880086737325", "-117489206105111877901926548", "128872288419083705914038986", "130736458515802378822355935", "-121556241169153639422095686", "72946657503235618393823866", "-125663836339334145024409857" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 1 ], "DATA": [ "-140434705281044213047311138", "-28116089490630336228630975", "103267266309928582800251241", "-109508303463489441066925072", "-37447408761258714290948079", "-48348712769317916926342261", "-140369639756363883929708964" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-9728758914128859071321834279", "33309572658775989148537936017", "-24668259330599725242298254275", "32381561123351360763354670731", "27825802768031595606954105738", "12938216164626281948188731232", "38931244077102153608387825162" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1 ], "DATA": [ "10382809776177658355331432551", "5168110327267343534258764218", "-3267242975961960293350853731", "18434029108189696411322869404", "35091091873636227218260118505", "21108165552752916181263958022", "3661245166176387607640692752" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "7740613132371376696717495398418", "-7707941191956096108812155008476", "-577241601325072443290122836094", "600167824497967521284893595791", "3272067843081989787146120493930", "-1030145054800689456877805292349", "-3650234270988646330203349279059" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "7680286774308848437433168536793", "6647565199451142367949930365916", "-2999133161143853022620246013872", "8804348153856092578496854714407", "-4952285004482144846215389827781", "-5318507214755856895477853590188", "3057348036468872181146029422349" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "3851649300561843980682812308976", "-5882755312906144634290218148759", "8705902980707176681205204470899", "-7226642038551930671682828321801", "-437315629757074034698382119925", "-7001534311562577654291968216882", "-5286321524483593992727653959615" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-2529962123499222848219244545038324", "395962666703149974739931139610831", "57550468475234656216408472967844", "16338486361970318802019682813255", "-1828779660992541246630598587074810", "-1623395548149665302699111049897215", "-1192580008134980600742905854055406" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "1236661467304426090883166445264402", "-371744076376489711195615032257743", "2054398363329009304225110885914248", "2282254142354611780924986446846003", "1335098616305012872647529459955419", "-2333515867164879976023231383269539", "114078543021693938325750457951500" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "464104358696147522307181233407201836", "528236181487245695215532819816780192", "-49582519618663647096439258967309615", "179810448282591210237518073467287227", "641083621863607981646658838917064705", "-588820141403146033804863670906323881", "-209172547311926173294677521407025768" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "27559909638983289023355911880960481", "-225884217842248648518867386017704678", "179899274801825611492587037729547162", "-645804265572868735329412546507385081", "-182001003702797629603891592835717884", "-196383748000158684449875627005718638", "168348539043763218469877957630344248" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "117107436058598227102369350940703879876", "109883501439450473736462013508285281794", "-101387654941698733435373306911208832174", "120539446302766831060730844387212578122", "36010116834354981290778572290676615335", "70903063265292543021168853120103056998", "110744444106697685492892984617486125322" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "139776289567340167051908795098181848110", "24016711019305619625037775723051412673", "151016471675979627009310128091857241769", "136878330327275804538678500765431659719", "-8175489881016674742304549992648258102", "25386838486740756955389088300546155350", "-58709249744971886150026166430504011518" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-156704989705626079171987231966068215654", "144772934450911580494762210707331833351", "-62369821560188986732383850394596682312", "80247096547059282305305172602938473045", "10035303749721112586071998129175567777", "92716485473901404146553068374146350189", "161388554243068610590428401872902762267" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-17578", "-532", "-29058", "3924", "-4368", "19542", "-9728", "-30043", "8547", "-22611" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1, 1, 0, 0 ], "DATA": [ "-8228", "28186", "27587", "1067", "-14722", "-4268", "19776", "32332", "24421", "-18365" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-1997107", "1099651", "-6097698", "-5444647", "2942038", "6725671", "-1283058", "-2877113", "7395450", "-3898388" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "3550186", "1817302", "2814668", "-2514642", "-135513", "5746437", "5761493", "-2082501", "-2006387", "-32607" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-1478559549", "-1775015351", "1327820002", "2075513820", "535530878", "1674041100", "38780695", "-618881922", "-1971347341", "271829544" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "-140335353", "1487240277", "232889439", "-1753621058", "-1650070658", "-1744522644", "445853726", "1638982616", "-2109491538", "-2031080162" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-316389553", "625297203", "-876965397", "-450718571", "943406065", "1809117943", "-1839008717", "-502467791", "-1208669208", "-640379010" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-183748795627", "277910510532", "142611883754", "444329076374", "347496352230", "-310879611982", "-445269624502", "-110940203350", "414382789989", "-299514156855" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "219186267200", "-45309151531", "361805447006", "429814377541", "185451738517", "-10717952791", "-455894293535", "-331945266020", "434683169128", "-427092008228" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-35787970178921", "126503320146241", "-47160409383224", "59076874352542", "-54218256211143", "131768259093096", "-140694998895085", "139329340093293", "2946344253399", "-11352879480875" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-47865164294131", "18996772330627", "82776417283585", "-48069371769128", "121080667078025", "-69990715553720", "-100167359977899", "-117897159499581", "-37425616598022", "-61336459221473" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 1, 1, 0, 1 ], "DATA": [ "24962306301532", "1342549247965", "69591424177484", "20605092392111", "96006776802363", "103132673511125", "-74474597325207", "123891091031891", "64647602566462", "-33817908700461" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "-19023776449141216", "-16616980776723277", "3679087078886167", "667623252211990", "28848050655097494", "35358382636133989", "-5528487795104389", "-35321874926701485", "33150704456809249", "-35673170700881073" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "9170335166359676", "-29152376325887866", "27394714588707273", "-18771083725437163", "-22856196754531896", "30245430696795348", "-11339487304468022", "11795737209688712", "17769114732159794", "-31196892450377756" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-7484701409133991703", "446279878089310952", "-9094698745269615531", "-635203874310126070", "-3266768293641445962", "8803106083192049543", "-3447545194464331596", "-74015213448162434", "-3494919863524679499", "4736749662952258658" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-7862885775777111821", "-2258339050710865419", "8640626623613280261", "1725474079450247834", "718161694297051028", "8263883702397073425", "7878929139943328067", "4794616785419809038", "1846785796472594179", "1051708514867826862" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-290275855847029699526", "2324764544272157302715", "-172538644237854396534", "-1015287161642688964086", "1083861575526398116043", "390994735554610775082", "-1196474592645325657445", "1865761624558280233031", "-1180403655805921813793", "530628104087968783213" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "954362787050824981929", "-667973333297772026679", "-1003872443597233202904", "421871596895408794289", "-1020239034548744444861", "718583437320525106540", "-379642667164083852664", "-202130841394416991102", "-1679999734562681423770", "-1700441593695289764413" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-2355756035699002368979", "1788582608362609979068", "976621959978375116033", "551918345212846459001", "2038738001872254112254", "-104241764012041061597", "1815004322628837717025", "-366030095473004039888", "-212188636908643977409", "-2212197067753662412081" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-157029864053237435585940", "580349464450639935626054", "274606111559531706158496", "-271531107176514984144572", "-283271260947610926629996", "-102992963880630057053386", "-151438874243484297766280", "267231765639661935826287", "6412575179562556131083", "-447284995394712155520012" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "82122697814709630556960", "-173674944109113014109917", "-545487196009632239998847", "497347624990809943036106", "-403342273894709674210901", "-408545942001809410098345", "-596472526090909588749503", "-516275782103386775147265", "571758314045638582710655", "-260772039821490889738006" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "151997540068811973892699427", "75525965773514350955041445", "86334400095685499194704768", "-99446476458777770659737285", "-36773561228702583990719229", "114833233963970911445935291", "-53580285648134851613933202", "-107255936095104666984539508", "11411674730798920037840357", "144310491490477735136741915" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1, 0, 0, 1 ], "DATA": [ "111539513842329174253547474", "134891345910083773681115377", "4883280022715795599398831", "-93475601770663945655266004", "69463434873811940745671423", "123064229528351805775861869", "-150461278409905335393794684", "5582686657272910816484271", "-41856573658852576713916684", "138680097641830104297300861" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-151772029737772929726728403", "35642043181692385372706993", "22083653620753836019287236", "-132784292547512147409668783", "-68828393278683857783700197", "-76581151082179815290212905", "39692809847260519264925858", "100268541173636388889856795", "-84443541640950891574906466", "-56403642052264302619910023" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-28164201707581895339796457396", "18512437657026840590853158320", "-725631091722477987250581723", "12378738266464188323153378043", "1884164162561779726605459613", "27835210450137278935849561805", "-33957093897194026793294927588", "24705712105631658413881985230", "-20752699574766143719562108952", "-25390963174699799711871467610" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "-30515239169198036972672053279", "-16714522371588133629312659687", "20089473716367036753654096200", "-26078971629524511775014771330", "17501247017496478629908047849", "-2928637003025782311993801087", "21291695736216764696592795686", "31536521455601627910426594990", "23365283652803300319589541711", "-21352428429043964655987017640" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1, 0, 1, 0 ], "DATA": [ "-1871820427329445630900753275833", "3231762858097800568377706484053", "-7593147127830990171207926819911", "-1328646573449438418929234745911", "7929494242245932071338824047776", "-3966690046585197915967741713421", "-2665638134601027112326964326194", "9493723499691953230194922419753", "-192784910368186052857726341060", "968457352886861403532133623589" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ "1799593923874731933560217881204", "-4108892863880965470839888494269", "-9288585435136926537350897814111", "5657445890165360156029413482367", "9566538541825565997512161866226", "-8660816252467569077590355298227", "-8872018686223204958393278690220", "2747605566704311318475704614163", "-5898641587743100416084001786515", "-3297823611414655609450786579230" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 0, 0, 0, 0 ], "DATA": [ "3031062632211056878773391959697", "4539157404236993497036298560754", "-89011930344651313281485505381", "6329124475577017732572836547528", "10013839443292860323774055369057", "-1017595829075308067813401189014", "2867701530636012507846076321736", "5049199938873542782410309132373", "-8360248192335873845319826952230", "-8573205393401942663978510341713" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "1119504478489389158114715132398217", "-1196861483850018707503891203312216", "1577676525610019362542143335287170", "-828524918774712338254714242796201", "2195372295538823359262046660137804", "-2090592767226916350765817876441046", "1267778113117072331278680263282405", "1899343874503183337053685132483541", "-1547991409572962043085814686518526", "1902230290372078686946188304673897" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "693293489583149990962930554361914", "440506011505476370060528227243171", "-1325383072032732660105399745630800", "917607968990684241788700315356108", "1743560964081196656600523814562949", "-1632234260262962178728375599295415", "-1245497842582367367033445464435845", "-1845276210960982951119705348291628", "-852049233534078951923277630775327", "-1061875438802298101158411197338174" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "662874337110619540246196737572533840", "-646712275200560229865863897728322608", "378197263526199456522160076139313754", "-655367409488714240118815274020981441", "-117363023724417392064609064984193546", "379968998004796503715316062979365386", "581971463075521396879850081437709346", "486590909317284352288769404708849420", "339153058170383739672141264076027632", "368011261202186797510454029551864010" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "611471791995589932000290011287342073", "-238320595795246885451815237904493030", "129885954954066668429052045867305389", "-593881411984223406801058087690854990", "208716878349646304461338632058153168", "294341067601902100416283102151104072", "48828028516396476570316169415440949", "621606094524397617471379378110976418", "-24429954817222979379361646729750851", "174309609239946237584153302119176933" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "38764350872143269830409050492580474774", "-1783556143128429347444249923707622466", "-73796525965417826742774297120572978723", "130398981933491578275105091583750862109", "166691015795481448891415156743596861442", "137059374902539614554733085225910695585", "-52077108529588489247990938655971916018", "91346637728026911154761148563604037923", "33500089399138437200753844820194648392", "-129284621583923871181594899462237973804" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "155321932412457790454079866994618338188", "27813009907673742133398066472232685666", "39272698207745895563541959640721556926", "-122722120683994266643963991712613165291", "-130174397926461656752602397741063391588", "82090308774368119635497681294792018586", "-59839129775569455414641041372080794175", "-24104505156877231797915313008748415591", "-108881215793562491957481808068270876072", "76694630138799355396392615062949976885" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0, 1, 0, 0 ], "DATA": [ "165547688447137835687250676649889094237", "-163339935116527850906013137286281712121", "-71022445758504577564847869973739799734", "145194176986878243647893329429477538354", "64521796946193208156404560040536806244", "39300237731247748947718794529964696310", "-57247755378136552231880640476421831435", "-46273099289129264614986219324959153834", "-55042141398285192896194894621330572941", "-152341554718452744063215754531967241888" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1 ], "DATA": [ "18698", "5226", "22280", "556", "8799", "29440", "451" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-23904", "31878", "21043", "7840", "-4061", "973", "-1228" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 0 ], "DATA": [ "2724325", "-4798020", "1908664", "-1806352", "-6499685", "-7280271", "-2257879" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "-4011001", "6863924", "-6697732", "-2217484", "1921902", "4699368", "5306140" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-121080047", "1970737025", "177569772", "-675421071", "-412793147", "1025987892", "-1489708604" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-2045004499", "539874307", "-1997427023", "1708485386", "541003597", "-2142889980", "-1571929387" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "-1277841834", "369437735", "115131390", "1767292952", "-737609351", "-1060380702", "1495387052" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "197387161100", "-254436366157", "286370837399", "19374886452", "-98965142936", "-44474737185", "443205205790" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 0, 1 ], "DATA": [ "-11626723519", "446888208290", "385477838312", "269092016708", "-115706867693", "186662180742", "443402603648" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-101903913010014", "-17846116772043", "-127568468992217", "138034513755219", "-22440181460716", "63908344819874", "71092752187712" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ "-10514368675102", "18576548276665", "11989898233479", "-94964748076982", "-25130417074236", "-12150591292188", "29627314247145" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-128065487316393", "-51607804296775", "9339364520761", "136634838298533", "-51187175560258", "120582565367596", "-113392529877605" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-6014210637334865", "8607066096461461", "24796296381889212", "-21982938527186312", "22830978621256619", "-26845542068057233", "-1207835268472696" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "3833038092952487", "22879010815516439", "35249449760784991", "-17068864041186991", "-26423231151454707", "-3215687274393412", "-23458194802999605" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 0 ], "DATA": [ "3125947421774122944", "-8281228069349294860", "-8475285212132358531", "-5284781313458999470", "8164712809196122859", "1812915631649193077", "4405900031418971351" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "4032410419461492356", "-3461877490707766960", "-1034170339182657409", "-3390159984142309139", "-8014068544953344330", "-2322898039699171514", "-4866093356373866447" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "746866223721616048807", "-1092102356202329139788", "-1546383537243345586308", "-147480023935690463533", "-1613816966398143873591", "-1366145607207253623527", "-1277834349517264956600" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-779628592648894155798", "833551847015048377342", "2350685094213395401616", "-646450799987582052640", "512671457318054258029", "889600361052664942457", "-1281226453859217419451" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-189794651087146365523", "1730990859510508481978", "84730118226610466482", "-2271149966448749710655", "-457693344089241611732", "220532349183084708104", "667161504208011399088" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-84007006749190537913356", "-246149963758082151700698", "58599320110198036377337", "419084729775542661330649", "-414234870389293282071427", "-525262946081080156584518", "136640685388491267545797" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-119294228731180826417958", "8761229936388021822082", "487012390130773891175460", "-311845427865456314818185", "559648900054229910709345", "139393885514427519830624", "107325890167816018081235" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "17089147827907561627837757", "-94429323577754654200100992", "-4473969571146714207261485", "141782659389643845693155915", "-93892358423502207196670410", "-145554766670598055216282741", "37597111607123600576198257" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-137314033837313163998637", "72449797169876132616353909", "-70161038289080296659386196", "-43462961961878972425525973", "42101201201192529980096415", "-81066870591720969078480725", "90267013220354821004402329" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "117713797316457619471160158", "-70167250212359158255102362", "-11552912460279859107138583", "-29885413874861556795770843", "4421888524449146349137592", "104643958570092735443054627", "50425644614078606322832703" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-30334928923978531796147629088", "27288518731802156666631771366", "13824378221071923474948637231", "4644693338626225213907016302", "11634130769035763812486389217", "7684952376163662025235478239", "-33239678106944925691053930083" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 1 ], "DATA": [ "-11528961365515593522622576679", "17133737086982694882922251878", "13646045761710072112428841241", "-11389171932161374967609239435", "4257823513423339355354889471", "-28413425236629841784233274651", "39588941214204919522713866880" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0 ], "DATA": [ "1793447129255862870401371387102", "-4521183177824322286238857131690", "-7015296797702626396918062112070", "-8405199754772912051218827051767", "-5847103288844519371863856372805", "-4986004223819988818141294208919", "1152074979591001029103861569975" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-6349141188765631755738960028456", "-2660433342897400293313691984408", "8110414928660755884872240488403", "-3255774704770639692268716192582", "-5624565439076989481811168712985", "8242854258628200994627074750006", "-632483613318590601476620619686" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-1906797773505486015084475661167", "1831532006604140257182189698277", "8658935091181123523496177509351", "4083753014008812543656172710815", "-703732616202838478277623562522", "-6578720101664337167243397144316", "-4231412865947716065230260553709" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "1481793740573226627257507227084301", "1987274798160197648897728495716203", "-2253699799337436551660654810527205", "-1664891149611323142615003737883555", "-812147342830058889078154049835125", "1133591471991032412967586858920266", "373066807517996728955491798981682" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "1308500817719514731390159580539356", "-2472263601229082178453567330649492", "806886383595845079530117586003857", "1483914875922945517574389647107123", "-2447637035834905490061551060827517", "-2584877975554282277356767257919022", "1137264392772462522125091324886529" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-202694815737861835748583337438619141", "-567475487177382119320244018738653105", "240870396569462382448904278424042886", "565877554788860310777372508885337792", "-16105501605830402808140696954471983", "-608678904857721513901782286805392766", "89500838347412802178721155164713004" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-92477909222259270232496020800286668", "-253868659242284214237281896032894612", "-571911363839430504720309156797127441", "-630228046136182620305988015405040527", "-145796823877333917115220895453473596", "-544416500996245714271727684022145559", "62108147716790867959393946884530570" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0 ], "DATA": [ "30196559748662473034683824360446700718", "-145993220223129636351442638734923612124", "-31744581905865529412669045728747964315", "-42689243253906548960505899054623799466", "-3278727297709430585641819106144408715", "-139194553088228069126405551246144950996", "-18860658848674927408656595634636715361" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "14098620813169356460637219212014489435", "-25133450166458778557679753986808320092", "-10717831626644290242659474112607023736", "61898419746062756560226489821314179510", "19338062857571632481395798424446979882", "-80563484496287165417026379634301213058", "38933787489872054716116934865976579056" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-18244737545312093171930563627683906325", "-69817450526652822027490267764416914485", "-161214969184468418385603069469725822547", "132041756256234747183020020523975575773", "-57875703366203729680850430132447338202", "133741584114580239596914015216689701177", "-104202051211600429556397759132978847898" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-25325", "10828", "-13355", "8467", "-14673", "13128", "-11338", "-7976", "-2935", "28799" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-25899", "15808", "5036", "5817", "21295", "6991", "30262", "-29206", "22545", "14281" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], "DATA": [ "5139488", "-4603800", "-4322358", "5568494", "2927252", "-2770864", "3950127", "-402389", "-1551328", "4307703" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0, 0, 0, 1 ], "DATA": [ "1273196", "-6043029", "-2124152", "6239939", "486272", "1367187", "-4466034", "-7300079", "1984393", "3056986" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "389208900", "185157703", "-2134187956", "217215497", "-2063962325", "284328714", "-1798439100", "-1345866522", "-44759387", "1568455956" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 0, 0, 0, 0, 0 ], "DATA": [ "-1988645525", "793006818", "-251491455", "247123050", "2046505050", "-1728516425", "-435424560", "-308364872", "-1892211705", "-126815551" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-773671966", "1358683392", "-1755115842", "1643669576", "-878872476", "-1809618284", "-1045892795", "-1775853987", "-1403818658", "482449875" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 0, 1 ], "DATA": [ "241099970526", "514993919499", "-496567360082", "502604322768", "-28679500586", "519917943945", "413447742457", "221064262119", "-391763012706", "-181222719579" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "509145597895", "47721898418", "201713081857", "183700101234", "314513786407", "-33405756748", "-532243541621", "-107548030858", "126151420688", "285956004348" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "116275668335152", "122141214739207", "-63430344564549", "-111832080367574", "112155501448142", "19656387304429", "96601821131959", "57545046918456", "107104563639101", "-59870112678790" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1, 1, 0, 0 ], "DATA": [ "45356471648224", "14779609611199", "-33962279108195", "-98944840181309", "91625511308220", "-110850158882954", "87916556152463", "80467617162481", "93925586648150", "-113372051519249" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "19606854898583", "-55504890190933", "-15147315545027", "40456424289866", "-6247604760181", "-93639696496745", "-84665878592107", "-100053850686284", "16367469522358", "99753371265059" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "24282640953032762", "10091709048435857", "-21974172647175281", "14073179053430222", "34109384946231056", "5226653906582893", "-33819520810414763", "-22931875185563773", "-24002071163260726", "-1715146803898567" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ "-30659716847381795", "-15836145091878170", "-30830890473364625", "-25423275503125707", "-15385992609067487", "-27328827163588642", "-26765399453321294", "21083654833354492", "-19809350790025647", "-2472980520466774" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "946065176298096897", "520206069634010822", "-2372009740528541735", "-3605114825675420401", "-4132067089809310737", "705141097139262042", "-7256635106430568358", "-8105301464259245359", "599492648809263584", "4365466939375502140" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "8277129333648717097", "-8448354335880876032", "2913319982453504823", "8736245116433424334", "6387017288276853607", "-4081208528945745050", "46515463897966715", "-1495751169585677679", "-6665852657838015529", "1497421528845365759" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 ], "DATA": [ "867602266070578599219", "2247746875415261031076", "1934169958376882247885", "-1983965311174074790761", "994286831208120914465", "-2053360980630830501597", "-352322923068724990708", "-275529843474582648430", "117569307044473834744", "-2057119187357998311386" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-983989144481816608421", "-227041553316359265394", "-2338398780830635151885", "-1425288624511611388041", "-1200460011151461084288", "-37860255133068080397", "168815163262920902402", "-1589043880708207878151", "1999861056072554689170", "-1063012832871660661701" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-934629893504979755792", "1330023560256145633360", "471809371618383733343", "2321922756694883212096", "-1145621103195820184004", "432986056770166636184", "527912618672815906694", "-1383597906408951487454", "-1840681975964590350984", "1003431919766458755951" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 1, 1, 0 ], "DATA": [ "-295917082043583082770621", "-231343988449013068326631", "5312383381819468949750", "-139157207556611762942758", "-241655483652404463403757", "-512476477719184343596604", "599410803946810481042938", "67078249271685046714164", "409870524862445838681667", "-198986259949659924965247" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0, 1, 1, 1 ], "DATA": [ "-278030681373064235595749", "393693012596137044946808", "113099188405109327204683", "415775568903274402438459", "-42076558072135973151428", "-219378272136268135495431", "-582613624074409026792710", "191194427072366781075751", "-9835457748214809790370", "442340177819471571537141" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-125407723627032808185149189", "-127693057085441986254509099", "143049524315591523657757604", "66172074121241277336416638", "105043654403595832965542333", "88321001961584231003626866", "43148804784313926260363839", "33183564004779395803780452", "-145197017290377600759079842", "119167929014336096222821000" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "119039071826650750074654630", "44751775197664106490961349", "27864406843200354265918393", "15658656552850314500730985", "-8220185761405769037321341", "-83569483397839034074232516", "-15999238535942008358971638", "51943353221378823545548335", "124460320105770697840371762", "96188240730380539592133999" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "-23740546198484978874875407", "76636632473705803978381047", "-56311328440996031370265536", "-17243365633279424695862822", "-86114647089246738996946962", "77779733094924174077783969", "-154377399990419141152689913", "-58108558577026614888610010", "32011657995884029676678558", "-85849178273273349349099772" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "38268503482208460783689168776", "415498958203045688529744846", "-5469243189225899466540717631", "-27069951860473838447007129302", "2300819963695030297957821979", "27013864454418030545712879052", "33795947339603501913179687623", "-36867304093401804903210992872", "6652565972383263112675657430", "21252184018008067722844465790" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "20488449679781423383625400864", "28945181199727711698700316165", "9535784301505998719738045771", "-20854663557065706429017034430", "-8496482734431467304571929654", "39529776626237782521053868212", "20282963743253661057874523422", "39168299995389261713687626653", "-2895774161826797358583901454", "27415786790395091445855072594" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "7274372610033635352026015117499", "-9390827953122518631957223639932", "-2203836129140192889937266167905", "4596600214642310892345897847127", "983389576577548413649282888500", "1964366100910371172669585857215", "-4168439792076922929733693975481", "3161618514384549987863457101037", "-7790441321891779327083714442512", "-7686467965572840837470847326318" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "9529887427430029315427723731097", "4336235122315854607986015683663", "-9706927952229870397920454025649", "4499727370692865250957373478909", "-5923209179547863758996820315917", "-2626628904315332468993978106025", "685916912461697873229704570250", "-2838726730006480465962589525945", "2039415276453860859795422056351", "7628942068105897131623029254615" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1, 0, 0, 0 ], "DATA": [ "126535036553237060539012806767", "-6939129554517585745715778319177", "-7532363115381746277033981571367", "7225306379513685172919544982624", "2081180225728056445004841883009", "6331729734153410979163746567493", "5450234392691226980537948053828", "-3339385174687947070096507831241", "-6555844461598841325077996060011", "9279765521180487317778800396224" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "2228023644252832655241598402411269", "431676701798775425411616196133933", "478260014584588677437872352249827", "-1507262382979849604564706747863757", "-2288462503836870608309925185139281", "49393515288688670462360817470707", "-2579210515571147629042989663475866", "589507628712335934139197077166366", "1989953354399049606431153717140929", "-965660095005622252483438158567899" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 0, 1, 1 ], "DATA": [ "-1816053260927088757790173812582767", "-2405480415192625953224616207989887", "-2328044646297041482107281923018610", "872987268389289710890938214776251", "643667272856490093246696929059356", "2135937810867387131247124235309141", "-348035586025203024239919226158367", "1200248021086499011518529585918610", "2026575827504135717840864217744664", "-1601765059510950727183880983778417" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "65074355550728584791094991091725510", "655697096560546628903998665713057646", "-526498334932676728875521670389483820", "-660406393116184719913940975963998661", "198613257448920495951253934679652556", "-43713779768026581579783258854735948", "24173852238556377483338028432771484", "-323536680150552559099100992362365559", "216972493779019710120712154999597849", "391545022377807490738436750954965632" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ "-128389779235154808800484297421230508", "491001613177916842185576260455747135", "-455096123184868191344797938977008190", "-309905590996917800177765369517052165", "349603248086460094879044594595289453", "173998676594990878495537914769309496", "580673547960002977806631871571859052", "304595872857331938936691935503967494", "-309341788503071907892541600073298273", "-445388828339388242207829894494395187" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0, 1, 1, 0 ], "DATA": [ "-116759118799696740987833268006312595507", "-240339187668656185170580579099921345", "-165524819667082744814917029626210589845", "112546391769223555027822608825303955489", "-48744565225770459292490161985480679384", "-61852298378918957927732680541183991273", "-109815699785003222394734941309206527496", "-161541054014304007303713671841308424373", "38232801342956757143716333070831819830", "-82876982197717752657894986064058509459" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "166626734583125926562909642736739648863", "-537561525487190105228039554746904245", "56724835995658384592515263322742572769", "154738535117659769768747849261742373987", "133841382783265265669107205599295242333", "-82481830171752743723415208240895428708", "47401574357301211579457100619126454320", "101450166334449946734731337888765885526", "-9655734409491446777009001964705514650", "-20474375954705981387014226963404713903" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0, 1, 0, 0 ], "DATA": [ "97166129096528442889485736329241954705", "-20654655171391656743269829102430663484", "-19774868324035406536117826166021377349", "-14595360169302341242602534087132194253", "-77171889502077963021661942682977180668", "-67805573373674104708253116675406619851", "107903125463174461399865488130215282736", "13666179504242759743615908229099766154", "-14923042555307260112864102940917241528", "-93735255641317514829426639104109826896" ] } ] }, { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-15637", "-21826", "23207", "24299", "9996", "18466", "7067" ] }, { "name": "f1", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 1 ], "DATA": [ "-26781", "22205", "-30862", "4191", "22380", "-31397", "-16113" ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ "-5606512", "-841159", "-1722865", "-7412802", "3421093", "1404395", "1763678" ] }, { "name": "f3", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-4174091", "-6797376", "-6735178", "3654374", "433672", "1994930", "1820130" ] }, { "name": "f4", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ "-27695193", "235417726", "1925960030", "-1068170333", "-1618156350", "-317495919", "1423560212" ] }, { "name": "f5", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ "-613673600", "875021835", "1453808997", "1717180769", "1580762908", "1840132086", "-1345743252" ] }, { "name": "f6", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "582856351", "1284574124", "340320670", "-1092859214", "1110297876", "1357226248", "1402761903" ] }, { "name": "f7", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0 ], "DATA": [ "-3179119623", "450280234735", "-246304555798", "-119827255352", "-525274364087", "-230257922796", "256828899002" ] }, { "name": "f8", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0 ], "DATA": [ "172087714641", "446734015461", "215626082740", "310099426632", "262400844124", "-403709578552", "-543955348408" ] }, { "name": "f9", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "79497412466729", "-17627606697345", "-56306219857785", "-44419610515947", "-10505875477448", "-137201986324719", "-41490578530344" ] }, { "name": "f10", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1 ], "DATA": [ "87785849879213", "72153641766797", "81344597122488", "116608220424101", "120127927299953", "91392864468246", "789277742054" ] }, { "name": "f11", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 1 ], "DATA": [ "16742530979330", "-27276173304601", "112787046376927", "80422014882468", "-53996799818821", "-51482063936845", "-109285996873057" ] }, { "name": "f12", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0 ], "DATA": [ "-27011971722513841", "-25715520475567957", "-31692200074647251", "19797016206092429", "13033529853072966", "-34190512535950112", "21391090647858827" ] }, { "name": "f13", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ "-2689551968658208", "-11978754119332225", "27943059826706703", "30925593314872225", "-25299415665884227", "-6588871840929638", "-13529612932491138" ] }, { "name": "f14", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ "-5877424150753455644", "-750157543599527648", "7844548247743823168", "1983390184771773250", "8109668973322170403", "4662100181419928346", "-7990733746417221683" ] }, { "name": "f15", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-4001316145812913855", "8636276014947384575", "255439123904346461", "-5910657878241983749", "-8574194995013797995", "6656912349600694116", "-153848629100633274" ] }, { "name": "f16", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 0, 0, 1 ], "DATA": [ "251402997439201416147", "-773260098260956584464", "-219271115207772204546", "-1036039623941759511644", "359482472533162316496", "1965913949245283018894", "599718122444765908168" ] }, { "name": "f17", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-1438319238194982234339", "1577759903359206474839", "188186678199824158300", "1276273466028897015455", "-1836059768603829828246", "-571809172339201000757", "-1423862618055329967277" ] }, { "name": "f18", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "1054493543273151908590", "-996798431306463901700", "1030912998214022157210", "1165909344715451535690", "1678758991439476296198", "-1261232816117225764157", "-275535923846867412400" ] }, { "name": "f19", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "DATA": [ "-265478781031709465954466", "-45885532800194657389599", "47123250342923949029333", "-28918098710716253013235", "-216245678184360771226258", "-67592927029058001825983", "-216606971904693341187546" ] }, { "name": "f20", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "-454834495768691198335625", "-121587423919595203596630", "-180702613619856867200053", "-586233612775783081876233", "-146313510918473805475897", "-19249352342706744822274", "602537232274909092052368" ] }, { "name": "f21", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-138817970110646913589715549", "71212874681781732508199238", "-36634575975569063090531384", "-118838660864234798642972931", "-63677435476346500972940088", "-143253302639820088137695781", "-10005431182625326908184063" ] }, { "name": "f22", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-24132845688739832600198664", "70223772618132296366621411", "39697053813238829872031989", "-4463405036654040634416246", "-114850114696918748689370315", "24197714526405477903473970", "-85613480517918645906993903" ] }, { "name": "f23", "count": 7, "VALIDITY": [ 0, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-78615821810256843179885264", "-9435616174371737446306696", "14724235985325181617831683", "-77554225473063349798518023", "-151687049843944286803049399", "152830772381840371720809534", "41342447912435620711044487" ] }, { "name": "f24", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ "-7452626331051061693783872103", "-37477886858762005988951878072", "29024564721311845466178480817", "27253762906471645828276940493", "-19788635697675024252549748007", "33227312025585737614263740454", "-37713444931406134334378281082" ] }, { "name": "f25", "count": 7, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 0 ], "DATA": [ "-25886791622624942457173736636", "34370751906575221175682318842", "-9932886803302690926326494218", "3324237332180497322466960837", "21125203591871926273778796597", "-11276967068824629033564949936", "28441370093039134290858059521" ] }, { "name": "f26", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 0 ], "DATA": [ "8912358001421025895832248632442", "-8916798266948379553115396290654", "-161154141964359329876263021833", "3858086459715378537257910522289", "8040021746182360241076098633516", "4716833692114616471008198034491", "3496285339125640710658712687020" ] }, { "name": "f27", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1 ], "DATA": [ "-9184049173255469554998225870729", "-1938356198264242814346866467899", "-4206826697895630793180053762195", "-9497517245200463036658297067684", "-5584086501913874255219443240156", "8412140854426525037463120473138", "-6042832331708829719042645761969" ] }, { "name": "f28", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 1 ], "DATA": [ "-6643586848533296755722567177838", "-1193404893030123421103724263326", "-7351808613718633597834537706852", "-4546872325918455995201693474980", "5044513193984398500316947717350", "-3564202300291159147721754279587", "9899326223400708541507662547701" ] }, { "name": "f29", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ "2538490507550203557654476537964592", "-171852815793316039580834325499002", "579673614387874228055248480157807", "1547316751704920851604992499434244", "82956532861960233155706332616871", "2270532599099227110701079190556481", "-1551256727215408667915785904404011" ] }, { "name": "f30", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-1697323961096836989990113960348683", "2269492088271616903476594268476027", "563384931675005850944800674271399", "1829983204109953557442975861814188", "-2099108441853345032440560303311823", "-1159901622227230738337097223322335", "1320399948477362419558701661162682" ] }, { "name": "f31", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ "-227649207549335409555288103485379868", "541380090188013381867176032991003854", "259040738101837697543772263422611758", "468607824887048977270254889483333053", "-37414275284716090078813150850969270", "414632729703080747958410749710577970", "-628951504936585588032719800558474149" ] }, { "name": "f32", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-585201720045860251224192209434417167", "468582923778969233979378862262684223", "134642897337460953327600517844593153", "36157897287467373309609536222700125", "42661449938741236454803304780412295", "212096786713676119368154634300273677", "585017229375111099843942749410082953" ] }, { "name": "f33", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0 ], "DATA": [ "-138591685930594567129919412699083355323", "77060413350994913049260494214760663038", "39700904836694418018645309917065762876", "-31664403115710800671259751306315806980", "63928998864039165846312590158916247833", "-3586180789611960233999050211148582684", "-42878742931457980860298222400389073217" ] }, { "name": "f34", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 1 ], "DATA": [ "-102158662182543310745650009612358481115", "-110662400233487451872440612242183030235", "-4734079005823009383561681338545373884", "25428205135817899705171309423993396942", "119935163858170485692044331984703633688", "27795979506302295159346827031425803891", "-17630802775799110077215936154218710903" ] }, { "name": "f35", "count": 7, "VALIDITY": [ 1, 0, 0, 1, 1, 0, 0 ], "DATA": [ "-167820728718740728757244841820454776505", "-95721966128938572947248972085749061555", "140529351854457382631585268999194010048", "-169071957026901707361219133333712549941", "-70939221559844798247914114191797953094", "-127614892614854677120213423457262257204", "-102794811166159877868688547277286760364" ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 0, 0, 1, 0, 0, 0 ], "DATA": [ "21474", "4094", "16234", "-10475", "-25906", "-12657", "13211", "20482", "29407", "-3604" ] }, { "name": "f1", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "9720", "-23643", "9540", "-23720", "31318", "-25292", "18536", "-9162", "58", "-7786" ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-7627572", "-38899", "2443108", "-438758", "-2210056", "2273095", "3114432", "-124491", "-1605021", "8009315" ] }, { "name": "f3", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "574992", "-1257531", "-2806474", "-3352205", "2915457", "-7014958", "-7579817", "-2215757", "-576189", "-8360149" ] }, { "name": "f4", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 1, 0, 1 ], "DATA": [ "-2041265396", "732584234", "-1999454822", "1704993828", "-210935878", "929286432", "1109525853", "582363395", "-1825179353", "1409680841" ] }, { "name": "f5", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ "1991977558", "1762056301", "1455216951", "1449185192", "254303971", "140252798", "935225387", "1899985780", "-927824213", "-208960670" ] }, { "name": "f6", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 1, 1, 0, 0, 0 ], "DATA": [ "1480625771", "1040969656", "-1003085364", "-553744456", "1343292649", "-325165443", "-1346706809", "-965803077", "723994410", "-20304136" ] }, { "name": "f7", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0, 0, 1, 0 ], "DATA": [ "-101818947634", "-229200189261", "-424288334493", "227262504132", "432555641000", "328751277087", "50785160524", "387280003539", "323702572325", "-130440892777" ] }, { "name": "f8", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 0, 0, 0, 1 ], "DATA": [ "285258640838", "-31233562378", "413535063358", "366447174903", "51794529119", "81104642011", "-229146846972", "-500253706209", "26427555726", "238540328738" ] }, { "name": "f9", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 1, 0, 1, 0, 1, 1 ], "DATA": [ "72984542244109", "-85349566014644", "-61750715247480", "-107963169778383", "-72064927212019", "74634683457234", "-5692709274261", "-137801733360366", "66863319822840", "53790391904569" ] }, { "name": "f10", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "-30506162400564", "44134792179061", "43208822949445", "116111507835088", "-52652763068808", "30227545743701", "54132435978158", "-84979416074353", "-77767629770125", "-49700250599015" ] }, { "name": "f11", "count": 10, "VALIDITY": [ 0, 1, 0, 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-33646284874231", "-39843156046137", "135855350625826", "67173686029441", "47373973603503", "-81315792116685", "126362930268105", "62326866041404", "-79917419899133", "116346318505097" ] }, { "name": "f12", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-17623564370335172", "13940102868728695", "31159892605228020", "19288139749976327", "-32162377831099808", "-25334882778439881", "-9293849274157177", "25753846620057242", "-2715500844487831", "17297835213572781" ] }, { "name": "f13", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ "-24802265556783267", "-1071851274851707", "-20708121978765252", "21106785059579409", "895452156504814", "-17439295017291261", "-29860734942289352", "-24366392879871907", "-24844781251317766", "5691042499081415" ] }, { "name": "f14", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 0, 0, 0, 0, 0 ], "DATA": [ "-950656153293958588", "-7026895800290111295", "-2906486337048393634", "-2487298974163906537", "-7240801780393004497", "-5928987398455158527", "91252511169998609", "-4434565223927059941", "6114237547594359253", "-5649241382570014194" ] }, { "name": "f15", "count": 10, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "-9125944744613843645", "2767912703851717499", "-5594414080371870722", "1326878363883364298", "6250061466047884457", "4167357360257738660", "3185444479792598034", "2850893211432323540", "4099400621617587983", "1911788567774553475" ] }, { "name": "f16", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "2243798170232923011194", "1260790852369524872646", "1926129514370913291084", "1923482934016917609948", "-1337597225338052576342", "-1396976913710878087379", "1760383914304461201041", "-2084424105474946896583", "546847183858239970168", "-950422829529740640046" ] }, { "name": "f17", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 0, 1, 0 ], "DATA": [ "2278561335636612407116", "2226551432527285815529", "1102614999141365247364", "282444931594177029536", "1707175857213323708724", "-1813457685182877716617", "1969563893365610364950", "-1294229647104229980192", "1058028980431521243786", "1425804620843155435443" ] }, { "name": "f18", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-666605901201678736091", "2294247840458377820087", "-120100005268248092109", "-2117030120296482991359", "-936538133650490467428", "1909556929153544496029", "668148430777958474013", "1300546260152428176254", "-1307847030795641403330", "962024918692270891085" ] }, { "name": "f19", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 0, 1, 0, 1 ], "DATA": [ "429007353965964812441246", "-539416061647283176800407", "-246119986866263363195168", "519288706159858079097852", "145290868494128799776491", "435391066127563014503225", "-212517380359649480302803", "229639965616877780412338", "-228549196983767607874750", "254322220652598819404701" ] }, { "name": "f20", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ "302744770146464834277007", "349890188007940826732505", "-395767415302716050551365", "-55506738483590549708985", "-42546607637771046763757", "599890205870578843548554", "-566080596849854240867084", "240819660280603920408014", "490355076012318604086830", "-444452608521493383189339" ] }, { "name": "f21", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 1, 0 ], "DATA": [ "140098423234498101984955781", "114972555620228418385387046", "-103855128687529946029391332", "-56453596465478079539961341", "112899417755966489798807822", "92479416756872049033014234", "-20220375538499957175667596", "49619147776130236286406382", "75868169169602577596893114", "-33991855971785669167684812" ] }, { "name": "f22", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 1, 1, 0, 1, 0, 0 ], "DATA": [ "-27955923849772806651093661", "-15134268062501921536631099", "-76180691187952624554132166", "8417609600056713913269682", "122210737333544046425988875", "-8255971110072598553761320", "72059712120402848706275333", "136669805857501664860962453", "117226067843670223572552823", "9538509353520989293188292" ] }, { "name": "f23", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 0, 1, 1 ], "DATA": [ "140326260020124618484882670", "-7604232162274599406456263", "-37578322955437058565218623", "82251242821235569071412696", "48934521009186759426554144", "-111502425314556830584865049", "143399285732546087389279250", "-145031030627834093577440455", "-56753812486185622944448939", "-131020484598181539722605940" ] }, { "name": "f24", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 0, 0 ], "DATA": [ "7163957691918374413440392122", "29895408949428293949825637565", "24953244787599080815469023689", "12933037616695046483291587011", "12911061711671395434197711654", "-662465023864870915737125277", "32902737578296234199724114196", "-13332149788556638691010135470", "28732116950622282825950924835", "-606082615276279539679739878" ] }, { "name": "f25", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 1, 0, 1, 1, 1 ], "DATA": [ "-2225441768919818543256115590", "36300525663713292309356899587", "22146605949936394498917396829", "-13799763611538096768550340984", "13697106267730772282848050012", "37987370502999491375497665834", "-15959212476158770181464939730", "37740896499376085097100094638", "-3277570747229261911544243044", "34646859779346958641044018113" ] }, { "name": "f26", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ "8464727847598438214667266343191", "3659092136516017614153405697408", "8898836896743628102604107262318", "-5124678781048461345216331096076", "7048310126754458455427182472252", "-7485149236907192851539148176189", "4765944987612723450221212104859", "3507852604123913333443796199231", "-3638691289771499903484086224708", "-1548294157076139554924261118715" ] }, { "name": "f27", "count": 10, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 0, 0, 1, 1 ], "DATA": [ "1960248254240505083559553622556", "6334649991475602141352059045232", "5429973568679996304865347312737", "-5574768356761641457841827808332", "2921634229104833962007853636995", "6645606916555794304028632924702", "7402711760974614198662134839264", "-2430215409038042282074825546118", "9189456250491239952337287624422", "-6746525342617036236303085693943" ] }, { "name": "f28", "count": 10, "VALIDITY": [ 1, 0, 0, 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ "-2303252604266546830450692395647", "24931561165839460953255704242", "3539774568243302781625620724843", "-4119755165665461319885896453094", "9980798001735069049033995442687", "6400453710768026204203981933498", "-1785561568444900500617881894975", "-1238657151175445676210732724686", "-4115303461735084227982174922353", "-4830151905286143893643036559970" ] }, { "name": "f29", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "DATA": [ "-744732514813865310675102410677837", "586923153993520038833541626305861", "631405070127015389249055779739635", "2375722591461493864257181946361349", "536327416123747764992062327728379", "4403250929897292075500020549198", "-920102429093431209270145990206427", "-1943957685901230324468842397328959", "-1793159792601568795672835117263773", "2539832186764726126109095059589925" ] }, { "name": "f30", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0 ], "DATA": [ "1736943855683500361344260536843977", "-1041915987085848668127887646443139", "-2073351044084729515377756324458994", "-2499860712727710930829417979309376", "-1334132038144948687896176896558215", "2497357047334342837216894037461981", "-2593640175489681036725320634980356", "83143781366050188466245498529346", "-1907502149335575039303599318337661", "769063293620504337906197233553371" ] }, { "name": "f31", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ "-250745229891973039581681177880223505", "327446904412556324074846927736374445", "-549111820347488138618064952270999012", "-107667493114547862984588921499141796", "-213442200532407781713670495439019398", "-357538486078878970354893736102694429", "556639003636036852879832060230162340", "-263609850789709040712893190708950606", "83507483892195138402414967192218438", "122845115500690774528208372788074092" ] }, { "name": "f32", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 0, 1 ], "DATA": [ "155941392458135251949747004728456096", "-351841600639304120253871824228008989", "13489132228213437891031310407260140", "98346439010414420839370528660894148", "110095662267504106346877402686062417", "272056686592458102197893958630607521", "-568455603353918918719268251254453391", "-641181026054843530064046004084619896", "-153598039697896424251301260354494095", "-238169729901866634377216907979943085" ] }, { "name": "f33", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ "-76486506229583148953575426031378420083", "-25268246003138515244204194206040713884", "151680852047855885899827416129131343552", "-65868597846063019193194432937954865449", "-37353304147507335221409678836410716004", "-83375093114704443137943855826794265870", "58765750477038687599983055893883073980", "-124247461021880357230108144628851373656", "-97168006373071667635001937044389371121", "147332252855265140513988431868146207163" ] }, { "name": "f34", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 1, 0, 1, 0, 1, 1 ], "DATA": [ "-93004622235567469002573559852323599733", "-136486146006693484453265708648174772555", "50333349154635707168287760155624850500", "56065748751155225550881020072304819143", "-89047403980608497862076968023015862559", "-138188932216419239057464674728657893212", "73805817848460887264703472962238324754", "-146887621381330685164768807944413153241", "-58194523976849290302949585080581301830", "-147081260537816177520155637200701099095" ] }, { "name": "f35", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 1, 0, 1, 1, 0, 0 ], "DATA": [ "-40110048976612650804598426597711067016", "124144731433352682852630636540845469156", "145245442856314145141703280651554450780", "-144190927886671777348422824785547904228", "51752695865288273345113477117624659857", "147665067149574600679327527424233686202", "51752916559949464310444304180016487848", "65187705658195518441807850839061980569", "-132615731354653260112463459795520015874", "-16458882186479214359093418885286436364" ] } ] } ] } ================================================ FILE: test/arrowjson/dictionary.json ================================================ { "schema": { "fields": [ { "name": "dict0", "type": { "name": "utf8" }, "nullable": true, "children": [], "dictionary": { "id": 0, "indexType": { "name": "int", "isSigned": true, "bitWidth": 8 }, "isOrdered": false } }, { "name": "dict1", "type": { "name": "utf8" }, "nullable": true, "children": [], "dictionary": { "id": 1, "indexType": { "name": "int", "isSigned": true, "bitWidth": 32 }, "isOrdered": false } }, { "name": "dict2", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": true, "children": [], "dictionary": { "id": 2, "indexType": { "name": "int", "isSigned": true, "bitWidth": 16 }, "isOrdered": false } } ] }, "dictionaries": [ { "id": 0, "data": { "count": 10, "columns": [ { "name": "DICT0", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 1, 1, 0, 0 ], "OFFSET": [ 0, 0, 0, 7, 7, 20, 27, 36, 45, 45, 45 ], "DATA": [ "", "", "kfii3eo", "", "\u00f4eh5\u77e2\u20ac\u00a3", "wa3fdmj", "3\u00b5i\u00a3146", "eo\u00f4rp3\u00b5", "", "" ] } ] } }, { "id": 1, "data": { "count": 5, "columns": [ { "name": "DICT1", "count": 5, "VALIDITY": [ 1, 0, 1, 0, 0 ], "OFFSET": [ 0, 11, 11, 20, 20, 20 ], "DATA": [ "n6\u20ac\u00b54g\u00b0", "", "o3\u77e2kbr2", "", "" ] } ] } }, { "id": 2, "data": { "count": 50, "columns": [ { "name": "DICT2", "count": 50, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1 ], "DATA": [ "-2147483648", "2147483647", "2063303031", "575556936", "994232893", "-733018692", "-1869839205", "818048194", "-780203826", "1956041779", "1352257839", "-411979330", "-1676682633", "-284358770", "-1409435091", "722395458", "2117191004", "-1448425086", "-1128563576", "-1190200669", "1780667813", "-987872595", "-160215270", "-2042614145", "-801319359", "940981753", "-1432601950", "-546534432", "970337383", "319587202", "494356019", "1832436202", "-932834552", "1463915583", "1779474803", "253308913", "1534215558", "1946313834", "87626535", "-2043979004", "1625504442", "-1819229860", "-176171291", "135039336", "-836907545", "792617942", "602157568", "81724808", "103096742", "282026629" ] } ] } } ], "batches": [ { "count": 7, "columns": [ { "name": "dict0", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 0 ], "DATA": [ 5, 9, 1, 8, 0, 8, 5 ] }, { "name": "dict1", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ 4, 2, 2, 3, 0, 0, 3 ] }, { "name": "dict2", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 1 ], "DATA": [ 8, 44, 46, 42, 11, 26, 15 ] } ] }, { "count": 10, "columns": [ { "name": "dict0", "count": 10, "VALIDITY": [ 0, 0, 1, 0, 1, 1, 0, 1, 1, 0 ], "DATA": [ 6, 8, 2, 3, 8, 4, 9, 2, 4, 8 ] }, { "name": "dict1", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 0, 0, 0, 0 ], "DATA": [ 4, 2, 3, 3, 4, 1, 3, 0, 2, 4 ] }, { "name": "dict2", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ 1, 38, 40, 32, 19, 46, 30, 25, 32, 28 ] } ] } ] } ================================================ FILE: test/arrowjson/dictionary_unsigned.json ================================================ { "schema": { "fields": [ { "name": "f0", "type": { "name": "utf8" }, "nullable": true, "children": [], "dictionary": { "id": 0, "indexType": { "name": "int", "isSigned": false, "bitWidth": 8 }, "isOrdered": false } }, { "name": "f1", "type": { "name": "utf8" }, "nullable": true, "children": [], "dictionary": { "id": 1, "indexType": { "name": "int", "isSigned": false, "bitWidth": 16 }, "isOrdered": false } }, { "name": "f2", "type": { "name": "utf8" }, "nullable": true, "children": [], "dictionary": { "id": 2, "indexType": { "name": "int", "isSigned": false, "bitWidth": 32 }, "isOrdered": false } } ] }, "dictionaries": [ { "id": 0, "data": { "count": 5, "columns": [ { "name": "DICT0", "count": 5, "VALIDITY": [ 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 7, 17, 26, 37, 45 ], "DATA": [ "nai4kkd", "1\u00a3n\u00a3\u00a336", "2f\u77e2wei4", "f\u00c2g\u00b5\u00b5m\u00f4", "\u00b5r2bkd1" ] } ] } }, { "id": 1, "data": { "count": 5, "columns": [ { "name": "DICT1", "count": 5, "VALIDITY": [ 1, 0, 0, 1, 1 ], "OFFSET": [ 0, 7, 7, 7, 15, 23 ], "DATA": [ "bor21pg", "", "", "erohj\u00c2d", "5\u00b0jogf2" ] } ] } }, { "id": 2, "data": { "count": 5, "columns": [ { "name": "DICT2", "count": 5, "VALIDITY": [ 0, 1, 1, 1, 1 ], "OFFSET": [ 0, 0, 10, 18, 29, 39 ], "DATA": [ "", "\u77e2\u00a3efhcg", "lii\u00b5wwm", "1\u20ace2\u77e24c", "bio\u20ac\u00b04l" ] } ] } } ], "batches": [ { "count": 7, "columns": [ { "name": "f0", "count": 7, "VALIDITY": [ 0, 1, 0, 1, 1, 1, 0 ], "DATA": [ 3, 1, 1, 0, 4, 1, 0 ] }, { "name": "f1", "count": 7, "VALIDITY": [ 0, 0, 1, 0, 0, 1, 0 ], "DATA": [ 1, 2, 0, 2, 1, 0, 0 ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ 0, 3, 4, 3, 2, 0, 1 ] } ] }, { "count": 10, "columns": [ { "name": "f0", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 0, 0, 1 ], "DATA": [ 4, 3, 2, 2, 0, 2, 4, 2, 2, 0 ] }, { "name": "f1", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1, 1, 0, 0 ], "DATA": [ 1, 0, 4, 4, 4, 2, 4, 2, 2, 4 ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ 1, 1, 4, 0, 1, 1, 1, 4, 3, 2 ] } ] } ] } ================================================ FILE: test/arrowjson/map.json ================================================ { "schema": { "fields": [ { "name": "map_nullable", "type": { "name": "map", "keysSorted": false }, "nullable": true, "children": [ { "name": "entries", "type": { "name": "struct" }, "nullable": false, "children": [ { "name": "key", "type": { "name": "utf8" }, "nullable": false, "children": [] }, { "name": "value", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] } ] } ] } ] }, "batches": [ { "count": 7, "columns": [ { "name": "map_nullable", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "OFFSET": [ 0, 3, 3, 3, 3, 5, 6, 9 ], "children": [ { "name": "entries", "count": 9, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "children": [ { "name": "key", "count": 9, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 7, 18, 26, 35, 42, 50, 59, 67, 77 ], "DATA": [ "nk23ghl", "hr\u00b0rr\u77e2\u00b0", "b\u00b5reapd", "\u00a3\u00a3fprlo", "42lfc64", "ifcmf\u00b54", "mo\u00b51\u00b5gr", "alfkm\u00c2m", "r\u20acj333\u00f4" ] }, { "name": "value", "count": 9, "VALIDITY": [ 1, 0, 1, 0, 1, 1, 1, 0, 1 ], "DATA": [ -2147483648, 2147483647, -364117737, -1036293739, 595547911, -136984751, 1113642047, -174490757, 247058944 ] } ] } ] } ] }, { "count": 10, "columns": [ { "name": "map_nullable", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0, 0, 1, 1 ], "OFFSET": [ 0, 0, 3, 5, 5, 5, 9, 9, 9, 12, 12 ], "children": [ { "name": "entries", "count": 12, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "children": [ { "name": "key", "count": 12, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 7, 15, 23, 31, 41, 48, 57, 67, 76, 88, 95, 106 ], "DATA": [ "nii3ppw", "am3a\u00c2f4", "pc\u00a3heh3", "d\u00c2a6n5b", "\u00f4mf2\u20acc4", "r1jdf4r", "njo4\u00b0\u00b0k", "or\u20acm\u00f4lr", "em\u00f4i\u00f4ab", "\u00a3\u20acg\u00c2\u00b53a", "nfrim36", "\u00b0\u00c2b\u00f4a\u00b5w" ] }, { "name": "value", "count": 12, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0 ], "DATA": [ -2147483648, 2147483647, -296180340, -44923686, 2123774743, 1831761340, 1290652534, 68203103, 1896803149, -1837559303, -517631115, 289945925 ] } ] } ] } ] } ] } ================================================ FILE: test/arrowjson/nested.json ================================================ { "schema": { "fields": [ { "name": "list_nullable", "type": { "name": "list" }, "nullable": true, "children": [ { "name": "item", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] } ] }, { "name": "fixedsizelist_nullable", "type": { "name": "fixedsizelist", "listSize": 4 }, "nullable": true, "children": [ { "name": "item", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] } ] }, { "name": "struct_nullable", "type": { "name": "struct" }, "nullable": true, "children": [ { "name": "f1", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "f2", "type": { "name": "utf8" }, "nullable": true, "children": [] } ] } ] }, "batches": [ { "count": 7, "columns": [ { "name": "list_nullable", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 0, 1 ], "OFFSET": [ 0, 0, 4, 4, 4, 4, 4, 7 ], "children": [ { "name": "item", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "DATA": [ -2147483648, 2147483647, 1151956889, -381487027, 873948027, 1643402405, 1658614443 ] } ] }, { "name": "fixedsizelist_nullable", "count": 7, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1 ], "children": [ { "name": "item", "count": 28, "VALIDITY": [ 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1 ], "DATA": [ -2147483648, 2147483647, 1455417103, -386721267, 1200555928, 416671823, -426485775, 1024100580, 1264993422, -1001974859, 1688456056, 1130147884, -1020099019, 1848995454, 2101085138, 1609275375, 1517442455, -1648565699, 994186796, 2006991970, -620340903, -846162493, 1646215825, 468219400, 1580372269, -1426972627, 1957501966, 1539427720 ] } ] }, { "name": "struct_nullable", "count": 7, "VALIDITY": [ 1, 1, 0, 1, 1, 1, 1 ], "children": [ { "name": "f1", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 0, 0 ], "DATA": [ -2147483648, 2147483647, 531016632, 1805412611, -1194513600, -1267423429, 1000139479 ] }, { "name": "f2", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0 ], "OFFSET": [ 0, 9, 9, 9, 9, 21, 21, 21 ], "DATA": [ "dej\u00c2\u00a3pr", "", "", "", "i\u77e2j\u00a3\u00b0\u00b5m", "", "" ] } ] } ] }, { "count": 10, "columns": [ { "name": "list_nullable", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 0, 0, 1, 1, 0, 0 ], "OFFSET": [ 0, 2, 5, 9, 9, 9, 9, 11, 13, 13, 13 ], "children": [ { "name": "item", "count": 13, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1 ], "DATA": [ -2147483648, 2147483647, 248935155, -322392706, -115480117, -1828031254, -1766278277, 444059471, -805219796, 1061714770, 423821865, 1278080160, -1936049755 ] } ] }, { "name": "fixedsizelist_nullable", "count": 10, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 1, 1, 1, 0 ], "children": [ { "name": "item", "count": 40, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0 ], "DATA": [ -2147483648, 2147483647, -376399341, -1632957441, -2143340689, -159866261, 419032293, 1196087612, 784095846, -435963426, -574319754, 534646696, -444534026, 1281811213, -105831675, -23408446, 348235550, -1986936151, -1558484167, 941484009, 2144967535, 830399805, 31470129, -792586642, -29778291, 1445584989, -1344862682, -1973973713, -875795314, 556638175, 1282112437, -2054079892, -881420523, 1038384760, 846630800, -442527632, -975132694, 84102477, 1771796204, -153752454 ] } ] }, { "name": "struct_nullable", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 0, 1, 0, 1, 1, 1 ], "children": [ { "name": "f1", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 0, 1, 1, 1 ], "DATA": [ -2147483648, 2147483647, -1568585736, 1732529716, 1236302061, -232126550, -1554873925, 191321500, 570901234, -2063634007 ] }, { "name": "f2", "count": 10, "VALIDITY": [ 0, 0, 0, 1, 1, 1, 1, 0, 0, 1 ], "OFFSET": [ 0, 0, 0, 0, 9, 20, 28, 36, 36, 36, 46 ], "DATA": [ "", "", "", "h3\u00f46c\u00b0i", "opa\u20acdf\u20ac", "ncdcpe\u00f4", "4fwflo\u00b0", "", "", "pf41\u00c2\u77e24" ] } ] } ] } ] } ================================================ FILE: test/arrowjson/primitive-empty.json ================================================ { "schema": { "fields": [ { "name": "bool_nullable", "type": { "name": "bool" }, "nullable": true, "children": [] }, { "name": "bool_nonnullable", "type": { "name": "bool" }, "nullable": false, "children": [] }, { "name": "int8_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "int8_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "int16_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "int16_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "int32_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "int32_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "int64_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "int64_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "uint8_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "uint8_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "uint16_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "uint16_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "uint32_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "uint32_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "uint64_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "uint64_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "float32_nullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": true, "children": [] }, { "name": "float32_nonnullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": false, "children": [] }, { "name": "float64_nullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": true, "children": [] }, { "name": "float64_nonnullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": false, "children": [] }, { "name": "binary_nullable", "type": { "name": "binary" }, "nullable": true, "children": [] }, { "name": "binary_nonnullable", "type": { "name": "binary" }, "nullable": false, "children": [] }, { "name": "utf8_nullable", "type": { "name": "utf8" }, "nullable": true, "children": [] }, { "name": "utf8_nonnullable", "type": { "name": "utf8" }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_19_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_19_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_120_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_120_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": false, "children": [] } ] }, "batches": [ { "count": 0, "columns": [ { "name": "bool_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "bool_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "binary_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "binary_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "fixedsizebinary_19_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_19_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] } ] }, { "count": 0, "columns": [ { "name": "bool_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "bool_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "binary_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "binary_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "fixedsizebinary_19_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_19_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] } ] }, { "count": 0, "columns": [ { "name": "bool_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "bool_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "int64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint8_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint16_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "uint64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float32_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "float64_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "binary_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "binary_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "utf8_nonnullable", "count": 0, "VALIDITY": [], "OFFSET": [ 0 ], "DATA": [] }, { "name": "fixedsizebinary_19_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_19_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nullable", "count": 0, "VALIDITY": [], "DATA": [] }, { "name": "fixedsizebinary_120_nonnullable", "count": 0, "VALIDITY": [], "DATA": [] } ] } ] } ================================================ FILE: test/arrowjson/primitive.json ================================================ { "schema": { "fields": [ { "name": "bool_nullable", "type": { "name": "bool" }, "nullable": true, "children": [] }, { "name": "bool_nonnullable", "type": { "name": "bool" }, "nullable": false, "children": [] }, { "name": "int8_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "int8_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "int16_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "int16_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "int32_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "int32_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "int64_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "int64_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "uint8_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "uint8_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "uint16_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "uint16_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "uint32_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "uint32_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "uint64_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "uint64_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "float32_nullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": true, "children": [] }, { "name": "float32_nonnullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": false, "children": [] }, { "name": "float64_nullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": true, "children": [] }, { "name": "float64_nonnullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": false, "children": [] }, { "name": "binary_nullable", "type": { "name": "binary" }, "nullable": true, "children": [] }, { "name": "binary_nonnullable", "type": { "name": "binary" }, "nullable": false, "children": [] }, { "name": "utf8_nullable", "type": { "name": "utf8" }, "nullable": true, "children": [] }, { "name": "utf8_nonnullable", "type": { "name": "utf8" }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_19_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_19_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_120_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_120_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": false, "children": [] } ] }, "batches": [ { "count": 7, "columns": [ { "name": "bool_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ true, false, true, true, true, true, true ] }, { "name": "bool_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ true, true, false, false, true, true, true ] }, { "name": "int8_nullable", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1 ], "DATA": [ -128, 127, -122, 28, -83, 84, 6 ] }, { "name": "int8_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -128, 127, 70, -103, 82, -1, 32 ] }, { "name": "int16_nullable", "count": 7, "VALIDITY": [ 1, 1, 0, 0, 1, 0, 0 ], "DATA": [ -32768, 32767, -28794, -4798, -9144, -6471, -28809 ] }, { "name": "int16_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -32768, 32767, -29297, -8851, -7684, 22714, 16282 ] }, { "name": "int32_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 0, 1, 1, 0 ], "DATA": [ -2147483648, 2147483647, -279511779, 789939398, 1566952883, 459144944, 112645645 ] }, { "name": "int32_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -2147483648, 2147483647, 1460305573, -781342537, -894485209, -611261236, -1733393626 ] }, { "name": "int64_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-2147483648", "2147483647", "-1338619198", "1195898730", "-196833435", "65722453", "227331787" ] }, { "name": "int64_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-2147483648", "2147483647", "134673412", "-1352599527", "799755210", "-1393335016", "-1401944839" ] }, { "name": "uint8_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 0 ], "DATA": [ 0, 255, 120, 36, 174, 74, 109 ] }, { "name": "uint8_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 255, 0, 157, 120, 198, 153 ] }, { "name": "uint16_nullable", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 1, 1 ], "DATA": [ 0, 65535, 58782, 40076, 39160, 7864, 28844 ] }, { "name": "uint16_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 65535, 19124, 52442, 22182, 56048, 17204 ] }, { "name": "uint32_nullable", "count": 7, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1 ], "DATA": [ 0, 2147483647, 919579300, 484217231, 723422877, 219335907, 1153201262 ] }, { "name": "uint32_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 2147483647, 1433308719, 68323683, 1195265875, 2106315800, 348904267 ] }, { "name": "uint64_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 1 ], "DATA": [ "0", "2147483647", "731435646", "90213947", "1630338070", "1558676470", "1207455205" ] }, { "name": "uint64_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "0", "2147483647", "1521875652", "1531293596", "277775570", "618952130", "1948103139" ] }, { "name": "float32_nullable", "count": 7, "VALIDITY": [ 1, 0, 1, 0, 0, 1, 0 ], "DATA": [ 138.786, 335.978, 868.094, -252.928, 433.858, -1044.852, -540.461 ] }, { "name": "float32_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -1645.756, -1634.941, -469.511, 489.177, 89.502, -1068.882, -1212.052 ] }, { "name": "float64_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ 915.264, 45.397, 1247.676, 724.292, -78.253, -751.91, 1169.159 ] }, { "name": "float64_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -404.256, -348.975, 273.1, 691.512, -448.703, 632.765, -1195.87 ] }, { "name": "binary_nullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 0 ], "OFFSET": [ 0, 0, 5, 7, 8, 14, 14, 14 ], "DATA": [ "", "51D99B869B", "F442", "83", "7C2BEBE65E6C", "", "" ] }, { "name": "binary_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 10, 14, 15, 16, 17, 26, 26 ], "DATA": [ "D6DDC948702117DC0840", "533708DF", "FB", "EA", "44", "BCCD4C61F06C883F73", "" ] }, { "name": "utf8_nullable", "count": 7, "VALIDITY": [ 1, 0, 0, 0, 1, 1, 1 ], "OFFSET": [ 0, 12, 12, 12, 12, 20, 27, 35 ], "DATA": [ "\u00b0\u00c2mgw\u20ac\u00b0", "", "", "", "iah6c\u00a36", "rokidwr", "6ja\u00b5fa1" ] }, { "name": "utf8_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 10, 19, 28, 36, 44, 52, 59 ], "DATA": [ "g\u77e2\u00a3k4c2", "orfbb\u77e2a", "m2k23\u00a3\u00b5", "5rl6\u00f45d", "egijr\u00b0b", "jple\u00b0ch", "rkiw42p" ] }, { "name": "fixedsizebinary_19_nullable", "count": 7, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ "50243F2281B31338EF651FCF9256388A319101", "9BECDDC65F62CFAB152F965E8C5311662E972F", "C7599811DDB43DFD35EAD3D3FB7FAF0AAFB9F2", "619D0F38B15A69CEA060657758CD9F6820C3DE", "C977E16BB93CFE804EB187E0DB4FB5C7E5E613", "2339A0EE888B27EFC693DF65DD717AB624DF0F", "F8F3629704844DAEF6E952A498504192D33913" ] }, { "name": "fixedsizebinary_19_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "9AD533F0DD44FC7229F465337FDAC9BEF5FACA", "E0EC91C117CC0C3E82BF9B288D7C3A7CB48A2E", "CC930B39BCAA159A659588751DC6EEA7F3F933", "2B236ECB372BF9606EC626C1CFA6928D7D0CE2", "DDF3F3C8D9DAA76406FCF6D6BEA375FF474C0E", "828A465C7EF01C9341782A7421C877718F2282", "FC34EC8BFEEF50D51F04B14DA64D1CD180757A" ] }, { "name": "fixedsizebinary_120_nullable", "count": 7, "VALIDITY": [ 0, 0, 0, 0, 1, 1, 0 ], "DATA": [ "8B96BE80C69B5CEA1C7535D822E8E8B93D22836318AD95BD33F2502084DF803C94735559F3191BB6D44DCE09BB77F9AF48E1851C0483B47A8EAE37ED92ABC071E46975F17685E9DD44640414E24F9C9F150BD29B0E2CABD4FB235A6D2CD5655D647724520C47AB1FE4DC7D418CDF3F55459E6DDF5B46F726", "ED0007F8435C9848C5F886245DC9C0AF5CA5796136146A967771E50DE82CA664B3C94C5AF0B10C8A6AB36B51D88E761181CEC6B8729FD6EB781255084F3A19F2CE1B0B2469B1E5B41468C125A391AED7313F8D088CB50C1B4090BA34BE96532377215DB8D02740C3577AEAB769CDC11F5C45EEFA54607F95", "E8253141328AE443B2DBC4217FFC251D7C8501924A54071DDEDF06D249F88119161DE1B4CFA929868E965E2EA102BBECA7908180C867F24E5CE1519DD734BC1E8EC77F12D9BF2876077B2A497E03B46B0203FB7BB9173B5A401B4A9D8DA152A13CC5AC218B0BED3131DE47F99909103754CB890104DCB4CD", "EE2CA14099A48D8E69A3EF6FBC20389F83F39692BD5E2490D25542EACB623FB79578CDD66C40A26B7732D1EFD5B5930E851F9E8C1E18C18455DA76D8BDB1562F7C6704F071B7707863AB91C9B3D10B5DB17BB451F3EF1FF1F1F2DD4AC2FB10FE6CB43E8F00EDDC5AF4785E7C4EED05BC5FEA73B9D08FDFB8", "277EB8E2CDCE71E471B8AF106BC682F76FA50F580EA09031E797D68464736EA29C50326845A935E1E8EFE77387B4DC4424DC1AA84834BB07779869BF50059953020034FF201E10CC710C76F87366435FD20AE6292712BB4A746ED0B8D263D3BE2A8E8DE8AF54E0667D04B1F8B3147206C74BECFD010320FC", "784E3DF66A677FD1D5D30EC37B3A9D8E77D876D1FE23EB4EDA9B721CF62D0C1AACBAEE2A4AF953547E472AF6D5F78BAE10D6FA7A2ED19F0C3D716F9C955157245EA7BAED58F6473DD03B6891CAA7AD551B1847A5C196A98716782C4C8479F0ED5BF5E505F01C0A772FA1805E2E7A81BC1409AC4D33CA25C7", "51BD3890033C88951DE28101F9A5B90D77083C4342065B86B5D7057F0E07827E70F398DAD0CF60AEAF30CADF988F6CD096F1B58AC48DBB4928BB0831A10D6180F300CBE31AF3D8388C8AF52EEA5DD317A213DFE69F53516DD90A2EDC015B29E73C07ED04C2DABC2EB63C42C474D26DCC4CBAD785D39533D3" ] }, { "name": "fixedsizebinary_120_nonnullable", "count": 7, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "E6439C34B42D20E636A6263B095E98EE7BFE0B33DFFF009EA4B9A14E15E5C7898D0408BB4FB41E73E50A25DE7210FDAF36B1DC57ACD9516236DE6C2B1D6B22618EE24EE657C4DE5001CF05088BCBDFD3B9B1048329DF8A9DF6AAF7A4747461A11E1A425D9007217C9542B7F93DAA9766338C4276A5D00B79", "D721489C85BF19A80DB99F84C529B900677A8647FFB6704A33914E8A80448E50143E2F7805F54D52B4B5C6920D57023D1B7FC4B8C4720F8768A9455D35F28097709CB8BE7EED669F20FFF7D8A11903AD66C61F1FC4D3761A487010F8164A20C02F7381F08C335DF4EF28E7E07E8CD1956B73C3805728CCF7", "D29C90A96829D519E24EA6E91C59CA3484BCD7E3E7ABD9FEA6358355C67A6F1E903466D1DCCF99ED5205E19C0B3E744827B1F1F4BF1F1438F818482591EC4CED55A21B0F8FE49E18E6CF8BC17920F62710F193A9D7CE786D4FFEA5AB2F86FC375B25239203F9CEF85144252D8D51D953A0AD549D82FF24E1", "665D643C48F33A6DC1C6D2A182F693BE49729C1976886AA5709E10B47992D4AC81682FD815447EC072852A112C5E056B21ACA45C4259DC6101893EF693CC83AB8023D431FD226C30DD9D543004D4655F18F1176B0A216FA425482244AB58E7B35C98E87A67D24723E17FB6009CD7520C1DDD6965F028CECF", "B6F739AA817BF09B1D324D2ED41CAC7BF2BB48DFA36539661EB015619365D109D56DD3B0FC0D6D1F88A3C14D20E130CE3F2AC7544983145B33D0F431556093958077967AC3420545A786C18F3C3C8A3358D5EB4EFE6829F4A7E989810578C9A1B2CEDC6A4154DE1138497E7EF69DCBCC96FC28DD50CAB9B2", "FF506B98BF6FB212E31A785A4B31BF897BBA4043C90FA9BE028183FB7A00B27055D4524EA9996A5CBC8C1A1EFC85D24A88385064245D9F694B504FCAB55A6DA2462FA3A0238990C3EBED78291272520D8C4DF6A8AB930562A0372C15CE02C89CF157819C8C8C1D39767BC413B64587EE41B95145B4C458E3", "EF0D024C1652A39F750EEEDC0C116634A844F238724C6825D30D228DA51CC7CFFA0F249B46A72B3A50C80593D070C91424625A738F810238EBE5DB26B3ED0D3358E3E762EDB538FB7A6CEBEDAA0EC2606061DC7AFD2F2707AC1EE64D25053BCE5351F15C297CC17382C81C1F6AEBBACBB54C6452E0B28F94" ] } ] }, { "count": 10, "columns": [ { "name": "bool_nullable", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 0, 1, 1, 1, 0 ], "DATA": [ false, false, false, true, true, false, true, true, true, false ] }, { "name": "bool_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ true, true, false, false, true, true, false, true, false, true ] }, { "name": "int8_nullable", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 1, 0, 0, 1, 1, 0 ], "DATA": [ -128, 127, -65, 89, -62, -22, -34, 65, 120, 15 ] }, { "name": "int8_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -128, 127, 123, 70, 100, -36, 113, -108, -6, 86 ] }, { "name": "int16_nullable", "count": 10, "VALIDITY": [ 1, 0, 0, 0, 0, 0, 0, 1, 0, 0 ], "DATA": [ -32768, 32767, 9740, -7733, 2637, -2516, -17569, 11730, -2908, -8593 ] }, { "name": "int16_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -32768, 32767, 11718, 15880, -6679, -22888, 23270, 18088, 20637, -15533 ] }, { "name": "int32_nullable", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ -2147483648, 2147483647, 477524758, -606134564, 393807314, -128112077, 717176459, -1600977792, 978954235, 1860414687 ] }, { "name": "int32_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -2147483648, 2147483647, -829757574, -578351845, -1348186787, 1890352675, 1086502698, 855304232, 181164543, 1057305604 ] }, { "name": "int64_nullable", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 1, 1, 1, 0, 1, 0 ], "DATA": [ "-2147483648", "2147483647", "794060722", "198709512", "-8995664", "-620701969", "1130202008", "415910253", "-84549711", "1106943323" ] }, { "name": "int64_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "-2147483648", "2147483647", "674480972", "1486723112", "633112390", "245208998", "-1174152026", "-321990725", "-902510995", "2067119255" ] }, { "name": "uint8_nullable", "count": 10, "VALIDITY": [ 1, 1, 0, 0, 0, 1, 1, 0, 1, 1 ], "DATA": [ 0, 255, 239, 62, 105, 206, 159, 158, 131, 187 ] }, { "name": "uint8_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 255, 228, 195, 119, 246, 233, 73, 227, 240 ] }, { "name": "uint16_nullable", "count": 10, "VALIDITY": [ 0, 1, 0, 1, 0, 1, 1, 1, 1, 1 ], "DATA": [ 0, 65535, 45066, 6178, 34711, 47964, 27512, 44887, 24908, 55701 ] }, { "name": "uint16_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 65535, 34339, 47690, 52236, 13, 16102, 11524, 57102, 29731 ] }, { "name": "uint32_nullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 0, 1, 1, 0, 0 ], "DATA": [ 0, 2147483647, 1647048552, 2065866351, 1252022470, 1848990303, 16726832, 597785424, 1240213577, 867494515 ] }, { "name": "uint32_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 0, 2147483647, 1852519148, 672549034, 1636451193, 2103372287, 680709579, 969728024, 959488127, 712543217 ] }, { "name": "uint64_nullable", "count": 10, "VALIDITY": [ 1, 1, 0, 1, 0, 0, 1, 1, 0, 0 ], "DATA": [ "0", "2147483647", "706989612", "89417634", "753617711", "863868960", "844763261", "1277577256", "520788201", "1438973022" ] }, { "name": "uint64_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "0", "2147483647", "343654149", "1874151572", "1405305125", "1695099154", "1823719899", "1099199355", "649954155", "1371446104" ] }, { "name": "float32_nullable", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 0, 1, 0, 0, 0, 0 ], "DATA": [ -1963.405, -521.792, -1054.599, -908.552, 478.058, 689.984, -728.344, -328.966, -992.419, 357.03 ] }, { "name": "float32_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ -3537.833, 887.934, -844.73, -564.775, -1264.466, 45.573, -1352.096, -1115.417, 62.628, 19.499 ] }, { "name": "float64_nullable", "count": 10, "VALIDITY": [ 0, 1, 1, 1, 0, 1, 0, 1, 1, 0 ], "DATA": [ 2290.58, 1382.481, 304.522, 919.074, -224.893, 941.403, 746.143, -945.379, -335.496, -92.736 ] }, { "name": "float64_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ 1537.239, 1200.259, 900.506, -183.952, 1331.255, 2447.965, -394.255, -1930.895, -85.89, -67.091 ] }, { "name": "binary_nullable", "count": 10, "VALIDITY": [ 0, 1, 1, 0, 0, 1, 0, 0, 1, 0 ], "OFFSET": [ 0, 0, 9, 11, 11, 11, 13, 13, 13, 13, 13 ], "DATA": [ "", "EA766EF95797A0C9B8", "84D5", "", "", "AD57", "", "", "", "" ] }, { "name": "binary_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 0, 1, 3, 4, 6, 7, 10, 12, 13, 14 ], "DATA": [ "", "3D", "9E86", "EA", "C091", "AD", "2D3828", "3EE2", "84", "F8" ] }, { "name": "utf8_nullable", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 1 ], "OFFSET": [ 0, 10, 10, 19, 28, 39, 39, 49, 59, 73, 83 ], "DATA": [ "f5\u77e2jnm\u00a3", "", "lnc\u77e2bbe", "a\u77e2r2pd1", "2d\u00f4\u00c2e\u20acg", "", "\u00b5\u00c2rh\u00c2rn", "g\u00b5e\u00f4h\u00f4r", "\u00a32\u77e2a\u77e2\u00b5\u00c2", "m\u00f4\u00f4\u00b56in" ] }, { "name": "utf8_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "OFFSET": [ 0, 10, 19, 26, 37, 46, 57, 65, 77, 86, 95 ], "DATA": [ "5j\u00a3n\u77e265", "pdwf\u00f4\u00f45", "ifwnble", "1g\u00c2\u00b0h\u00b0\u00c2", "4e\u00c2fkb\u00c2", "\u00a3\u20ac\u00b55omr", "4f36\u00a3rp", "jhw\u20ac\u20ac\u00f4c", "5r\u77e2wl2h", "6\u77e2bnin1" ] }, { "name": "fixedsizebinary_19_nullable", "count": 10, "VALIDITY": [ 0, 0, 1, 1, 1, 0, 1, 0, 0, 1 ], "DATA": [ "C85E25A3CCE5865762DCFC6E07E23481189581", "833BACCDA190F764DEDB416EE08B8BC302AB63", "D7573E24B94D3B70E23743918DBE9BA3272FA5", "E3AE2D60B3E7513863A71E78874BE0B714524A", "833E27429BC71A6CBCC0A1349F58112AA5307E", "CB3F3571BCA366F6DA6AD3B182452529BDEA1C", "5B4F8DD3A1694411902AAF6C0087143581B2CB", "D7C308DBC4A59CCD176076941788705CFDA2ED", "D5C307EEE12A6E4D0B2AD848DC52EB252ECDEF", "D778B9EB37E4D5FFC1BF753ABAEB069A5E45E2" ] }, { "name": "fixedsizebinary_19_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "C75F1645509DC011AB32AC2F576CAE7ECA7B16", "CB8E1A1F0A4B75B4E8E199EE6959E2C0CC64B6", "9D53CFDBD99B04F4FB034256CDDAAC86CD134A", "0B78B4D2F163FC63BF2F36FEC8E9466B79A3C8", "D89F13F6448B6E7FD84B2E19A460AD8E105E31", "CE7121C392A8560257CB0F36A3D4CEFEAA120D", "540FE8D61D90A7B5984B45A082BF2842126253", "97EC73B3236B90B522AAC15817626E0566780D", "3A8B546C8FE7624CD4E1EE8B3E0D77D89D5EC1", "3E601ED2EA586F69F56387556D17512A482A08" ] }, { "name": "fixedsizebinary_120_nullable", "count": 10, "VALIDITY": [ 1, 0, 1, 1, 1, 1, 0, 0, 1, 1 ], "DATA": [ "D49D31D22EB091811343B3B583AE276D5DF5BC8EA636000DE72FCBB3031FBA4D05F433EBC75124D60C56656F5411B3A0E6EECDA4346CF3105F87E7B1DEBF3C6FCF938F07B3C470DDE33436D19C07D9BA128B47554E5276D47C21C715BEF67E3802ECA55BB6759E50F130590ADEDDCD86A958F56C9D077C70", "6AC6110369AFA04648213B52CD109586E85CD77D5B3F262576BC3FF98A9EC90761912FC88940F798CDA6163F3E3D7994285A903470365243F551111621F6350B078CA055001EB406B5A7C07706F2E04B2730A2335D87C151368A160EDBC0C26201EA6441990354ED302894B8ACC1A257D6DE9E17B2564898", "DCE126B0DCB6195E93DC878CAF5F064BAE1184A6DBC06C2C8A1628D019BCEC17411521E3BC58A8ACBC73FCCE887B9D6E45645B7C2100AD624FF6734241676C747A3C6DC61C52D605A379F906C662215DCE3115A7BD1213B6E7F5C9AFF2D6221BFE4C0C4FF4AF9B18BA2E7D425BD8D98C0D7BDA249EA37BA8", "4B18DD4F1927AD7AF60F7C11C982FCCCFD42A17849EA8654A30DDC58C1CBD641C1B70A924EA1F3D2DCF750DBF87B16CBC4E00D9617E88737A22B9F9B7D20F779381A71879FC46D591A7C0426A2BC5FF563C6F33CA0566A6E7D471033961D45F685D403F4E9D3164FE8CF5ECBF131D3EFBA371DAE1C6C2E6B", "33C9DE972AD49D58C52F895B0F81EEE1E819BA427E9197F2DC3F37FD9D6B31E847016B4590BC72C60BA40C89D0647A0064C5192FD917CCBC7351650885DAA03217E8FFA1B568582AC2354499852CDCC17967F3301DE60EAB044A66AD0A0175DCE71229D5ECD83325B0D2A4D75169A38DC11F6504A8CF9F92", "943EA9C4927C15AB07594F7FABFAE47061670FF75663C9B3AF5F08C8EE1523AACA9E631996D3A17644B23DC1CC6FB3157A177340CA9F93FC7D0E2C3BF907EB74AB4C5DE8B49319B86BB9457649360DA39B66BC145721A6BAF056961172EC6C93CB7CBFC9A2C682DBE8A74F8CDC20324CDAB7388480908F18", "00FA841BCDD7C5D584C9C27E3577C7CC3FD1AC6C760B1914D1CBF765418EF21005AB7E065AF3C08424F7CEF98EECAE1694939728B71F750A096B3FF8F78843AC768C37FC7712A075786C7F1FC90AD316D20838693002DEA9B32B5864948709B200FF3E278B578E0B47A1E63583DAB18AABED963207642992", "4DCB937A9E3144A2383FFF5C3EF93EFFD128FC3C131DD9469A2CB0E59DA7EBDBA77AD01A0979B99FBA10055567EDA3CFCA77A515BD1D4D632C6BA9B698813F1C0AFFBEFF227D140407BF5728EC943F9D3833DA1B8D2B5B60398CB219500022B1AB9312273C5275D6282EC4A4E648B6491212B954470CA6BE", "A0BBC95D4894952D2E4945538AF4CE60828A1B94949143EEA065C0917BD5CDDBD68AA838AA1F75EFED37DC7256618C0CA762EE7CF75A287D529ADBFC81FD11A5F75CE23BF27940150E737E73992ACF085F52AE14E02BCEE2D229F20AEB862A8CC3D5881AE4E6267CC8B1EEF1D2AC97D4769CAD14DD4BFB96", "F54B84661C426B3D0E37A455738000E50EF024258EBCE2AA3FEC5AA4E2EC3C48150B56BD44E96658C7CCB6267DB8C5484DA153FBFE0478073720BCBB65927681B36C17AE976568129E3C2FD1CBBDE89DBEE1D461FD7C6DBA455D80DC02A107855C5D04808AA164D85CF1F72EDB153685B2046E12968DB052" ] }, { "name": "fixedsizebinary_120_nonnullable", "count": 10, "VALIDITY": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "DATA": [ "383BB0920976B030D92B51AA60DC74922A0EB112955D1D934E19EC0CDE12CAB87C6E069439EB4F9FC105F7250BFE7DD983CC10728B8445735A7E52720FBDCD4AB7113BF52B797A05B187F572A330DABEA5E8DB5698168538775825001E7EB6CA0468F06F623A77AD99E5BB63D90A267AF243D7840B099F2C", "074AC397B235ADA4A90B56B3B37D5148BEA22D6A6746B308DEA7E81A7294AEF6F3C6097A0941CA8DF01147B27040A672D5DCCF073DEB5DC0AC360E083BFB8378D7364EB2E4A09D629264BC9ED28D967184A6710424DC36D80E7D4FF499FA3CF1C0D77117CFB3E5783735F8B00D52A1243521A598F44A139F", "0612AE357E65463291258B15782DA04D8B8F004F4360EB1DAB17FB57B5A5772ECCF419EE132FC94A1C0D00A2C731B887C25CDAF192BE78E6C94145421884243B234DC7A80592E70D6ECCCAEA8C08AEAE2A1AD70D786B039DA894B6FE7996F5BF6EC6670FCBBE7042430D0174D5CA9043DB7B79899A360C94", "FE0A6A2B3A475AF6AA63CE24997BCAFA096B9269DA9368CEBD03444223F2497907FBD66CA071A7BEAA68D76CFD9A507280BF6108303F16558225883351E2FDF88E425DBE6ACEBE29E69CF57E41B74952391DF1B04F80363E3879A1326B9DD2D4AD94092D1FDD8EE574AC69D1F22A1BC1EC7F04BBAEC32AB7", "588F15533772FD3C26415F6DC5DDB98264BEB79868908B6CB216D59A3A227E75F443A448B0D7A405AA248F6A6F049EB247B1826A02DA6F2624E16B0A899056B1D0565429434006CB11D5E3D224CA4AE7A4C19817685CB46DC0BF789AC94BF755A47B4668EBA8898CF85CD0C4F2309E97F0FE25EFB508B635", "5351D21576971DFFDF48693B65895450FC098F21722B44E4CE583E969ACDF47A686AA849B265CECEAAD67CD122C10FBA7069E52326B64FD146054EE8C3F621AF28FFB2127E75FFC99B0E72B72BC615C4E9B7310C088CE5883CF41AC717DABCDF6F140B9933C831CB828EF9C02E02DC35501B527E46B1933C", "6B7D83AB9AC067A38C0F2A95A4B4A3CE63C4731C729C9ED568FDA35D750262B5E7B44809D131F9F7996B2630C6075C083F31736EBA9CBB1CC62A257EBE294C55B750154E004F768EB670F648E17EB5DA40E28350B19070E924001F16846F6BC197897F189E31F30C93D43098DBE39F0EBF2E4A7FD7A25A55", "26C342B09B0D6D690753743D15F78B9F2CDBAF14B95DF0D5BC9D7E1AC0C4416F3852AD0CD3D51610A6B7B6BEC766672E274C971E5692FA9AB982C3C828EA9A270BFB524EDCA50D0E8444540B82752201BDC9BA56CE6A01A04E25D5514C045515FDE12AE0B0A0533AAFD5E5FF18ABD22312DFEC854C496391", "7C6D76970A57EFEC88857971E84675A3EDC47F0B2E1AA20B916601842394BC674F3CF10729DF32D10BC7BE05B74B8F68E24884637F0553A7EC84353C43B4640AC5582EBAF0827D247CE49BC95B74D842E2EE25D80741F80C3019D7EDF58CCB47112C70835F32E538F3664B5625C5FC3C761FE96E329AF03C", "3A97DD53C812EBB096458C9B8E382F9404BB5F1F29FFF7973E15FB547A06F32DAFD300719AEE30E11DF794603B5D196AAAB5BE44A127590388E420367C3F542E6288A1505D83283B63417F9DBD5E2609D436773106940756DDB71910ACFD5FEF8A1F92AD3366E778FD1849C1CB2C0439B6D5719299E59D96" ] } ] } ] } ================================================ FILE: test/arrowjson/primitive_no_batches.json ================================================ { "schema": { "fields": [ { "name": "bool_nullable", "type": { "name": "bool" }, "nullable": true, "children": [] }, { "name": "bool_nonnullable", "type": { "name": "bool" }, "nullable": false, "children": [] }, { "name": "int8_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "int8_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "int16_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "int16_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "int32_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "int32_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "int64_nullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "int64_nonnullable", "type": { "name": "int", "isSigned": true, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "uint8_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": true, "children": [] }, { "name": "uint8_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 8 }, "nullable": false, "children": [] }, { "name": "uint16_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": true, "children": [] }, { "name": "uint16_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 16 }, "nullable": false, "children": [] }, { "name": "uint32_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": true, "children": [] }, { "name": "uint32_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 32 }, "nullable": false, "children": [] }, { "name": "uint64_nullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": true, "children": [] }, { "name": "uint64_nonnullable", "type": { "name": "int", "isSigned": false, "bitWidth": 64 }, "nullable": false, "children": [] }, { "name": "float32_nullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": true, "children": [] }, { "name": "float32_nonnullable", "type": { "name": "floatingpoint", "precision": "SINGLE" }, "nullable": false, "children": [] }, { "name": "float64_nullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": true, "children": [] }, { "name": "float64_nonnullable", "type": { "name": "floatingpoint", "precision": "DOUBLE" }, "nullable": false, "children": [] }, { "name": "binary_nullable", "type": { "name": "binary" }, "nullable": true, "children": [] }, { "name": "binary_nonnullable", "type": { "name": "binary" }, "nullable": false, "children": [] }, { "name": "utf8_nullable", "type": { "name": "utf8" }, "nullable": true, "children": [] }, { "name": "utf8_nonnullable", "type": { "name": "utf8" }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_19_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_19_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 19 }, "nullable": false, "children": [] }, { "name": "fixedsizebinary_120_nullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": true, "children": [] }, { "name": "fixedsizebinary_120_nonnullable", "type": { "name": "fixedsizebinary", "byteWidth": 120 }, "nullable": false, "children": [] } ] }, "batches": [] } ================================================ FILE: test/arrowjson.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. module ArrowJSON using Mmap using StructTypes, JSON3, Tables, SentinelArrays, Arrow # read json files as "table" # write to arrow stream/file # read arrow stream/file back abstract type Type end Type() = Null("null") StructTypes.StructType(::Base.Type{Type}) = StructTypes.AbstractType() children(::Base.Type{T}) where {T} = Field[] mutable struct Int <: Type name::String bitWidth::Int64 isSigned::Base.Bool end Int() = Int("", 0, true) Type(::Base.Type{T}) where {T<:Integer} = Int("int", 8 * sizeof(T), T <: Signed) StructTypes.StructType(::Base.Type{Int}) = StructTypes.Mutable() function juliatype(f, x::Int) T = x.bitWidth == 8 ? Int8 : x.bitWidth == 16 ? Int16 : x.bitWidth == 32 ? Int32 : x.bitWidth == 64 ? Int64 : Int128 return x.isSigned ? T : unsigned(T) end struct FloatingPoint <: Type name::String precision::String end Type(::Base.Type{T}) where {T<:AbstractFloat} = FloatingPoint( "floatingpoint", T == Float16 ? "HALF" : T == Float32 ? "SINGLE" : "DOUBLE", ) StructTypes.StructType(::Base.Type{FloatingPoint}) = StructTypes.Struct() juliatype(f, x::FloatingPoint) = x.precision == "HALF" ? Float16 : x.precision == "SINGLE" ? Float32 : Float64 struct FixedSizeBinary <: Type name::String byteWidth::Int64 end Type(::Base.Type{NTuple{N,UInt8}}) where {N} = FixedSizeBinary("fixedsizebinary", N) children(::Base.Type{NTuple{N,UInt8}}) where {N} = Field[] StructTypes.StructType(::Base.Type{FixedSizeBinary}) = StructTypes.Struct() juliatype(f, x::FixedSizeBinary) = NTuple{x.byteWidth,UInt8} struct Decimal <: Type name::String precision::Int32 scale::Int32 end Type(::Base.Type{Arrow.Decimal{P,S,T}}) where {P,S,T} = Decimal("decimal", P, S) StructTypes.StructType(::Base.Type{Decimal}) = StructTypes.Struct() juliatype(f, x::Decimal) = Arrow.Decimal{x.precision,x.scale,Int128} mutable struct Timestamp <: Type name::String unit::String timezone::Union{Nothing,String} end Timestamp() = Timestamp("", "", nothing) unit(U) = U == Arrow.Meta.TimeUnit.SECOND ? "SECOND" : U == Arrow.Meta.TimeUnit.MILLISECOND ? "MILLISECOND" : U == Arrow.Meta.TimeUnit.MICROSECOND ? "MICROSECOND" : "NANOSECOND" Type(::Base.Type{Arrow.Timestamp{U,TZ}}) where {U,TZ} = Timestamp("timestamp", unit(U), TZ === nothing ? nothing : String(TZ)) StructTypes.StructType(::Base.Type{Timestamp}) = StructTypes.Mutable() unitT(u) = u == "SECOND" ? Arrow.Meta.TimeUnit.SECOND : u == "MILLISECOND" ? Arrow.Meta.TimeUnit.MILLISECOND : u == "MICROSECOND" ? Arrow.Meta.TimeUnit.MICROSECOND : Arrow.Meta.TimeUnit.NANOSECOND juliatype(f, x::Timestamp) = Arrow.Timestamp{unitT(x.unit),x.timezone === nothing ? nothing : Symbol(x.timezone)} struct Duration <: Type name::String unit::String end Type(::Base.Type{Arrow.Duration{U}}) where {U} = Duration("duration", unit(U)) StructTypes.StructType(::Base.Type{Duration}) = StructTypes.Struct() juliatype(f, x::Duration) = Arrow.Duration{unit % (x.unit)} struct Date <: Type name::String unit::String end Type(::Base.Type{Arrow.Date{U,T}}) where {U,T} = Date("date", U == Arrow.Meta.DateUnit.DAY ? "DAY" : "MILLISECOND") StructTypes.StructType(::Base.Type{Date}) = StructTypes.Struct() juliatype(f, x::Date) = Arrow.Date{ x.unit == "DAY" ? Arrow.Meta.DateUnit.DAY : Arrow.Meta.DateUnit.MILLISECOND, x.unit == "DAY" ? Int32 : Int64, } struct Time <: Type name::String unit::String bitWidth::Int64 end Type(::Base.Type{Arrow.Time{U,T}}) where {U,T} = Time("time", unit(U), 8 * sizeof(T)) StructTypes.StructType(::Base.Type{Time}) = StructTypes.Struct() juliatype(f, x::Time) = Arrow.Time{unitT(x.unit),x.unit == "SECOND" || x.unit == "MILLISECOND" ? Int32 : Int64} struct Interval <: Type name::String unit::String end Type(::Base.Type{Arrow.Interval{U,T}}) where {U,T} = Interval( "interval", U == Arrow.Meta.IntervalUnit.YEAR_MONTH ? "YEAR_MONTH" : "DAY_TIME", ) StructTypes.StructType(::Base.Type{Interval}) = StructTypes.Struct() juliatype(f, x::Interval) = Arrow.Interval{ x.unit == "YEAR_MONTH" ? Arrow.Meta.IntervalUnit.YEAR_MONTH : Arrow.Meta.IntervalUnit.DAY_TIME, x.unit == "YEAR_MONTH" ? Int32 : Int64, } struct UnionT <: Type name::String mode::String typIds::Vector{Int64} end Type(::Base.Type{Arrow.UnionT{T,typeIds,U}}) where {T,typeIds,U} = UnionT("union", T == Arrow.Meta.UnionMode.Dense ? "DENSE" : "SPARSE", collect(typeIds)) children(::Base.Type{Arrow.UnionT{T,typeIds,U}}) where {T,typeIds,U} = Field[Field("", fieldtype(U, i), nothing) for i = 1:fieldcount(U)] StructTypes.StructType(::Base.Type{UnionT}) = StructTypes.Struct() juliatype(f, x::UnionT) = Arrow.UnionT{ x.mode == "DENSE" ? Arrow.Meta.UnionMode.DENSE : Arrow.Meta.UnionMode.SPARSE, Tuple(x.typeIds), Tuple{(juliatype(y) for y in f.children)...}, } struct List <: Type name::String end Type(::Base.Type{Vector{T}}) where {T} = List("list") children(::Base.Type{Vector{T}}) where {T} = [Field("item", T, nothing)] StructTypes.StructType(::Base.Type{List}) = StructTypes.Struct() juliatype(f, x::List) = Vector{juliatype(f.children[1])} struct LargeList <: Type name::String end StructTypes.StructType(::Base.Type{LargeList}) = StructTypes.Struct() juliatype(f, x::LargeList) = Vector{juliatype(f.children[1])} struct FixedSizeList <: Type name::String listSize::Int64 end Type(::Base.Type{NTuple{N,T}}) where {N,T} = FixedSizeList("fixedsizelist", N) children(::Base.Type{NTuple{N,T}}) where {N,T} = [Field("item", T, nothing)] StructTypes.StructType(::Base.Type{FixedSizeList}) = StructTypes.Struct() juliatype(f, x::FixedSizeList) = NTuple{x.listSize,juliatype(f.children[1])} struct Struct <: Type name::String end Type(::Base.Type{NamedTuple{names,types}}) where {names,types} = Struct("struct") children(::Base.Type{NamedTuple{names,types}}) where {names,types} = [Field(names[i], fieldtype(types, i), nothing) for i = 1:length(names)] StructTypes.StructType(::Base.Type{Struct}) = StructTypes.Struct() juliatype(f, x::Struct) = NamedTuple{ Tuple(Symbol(x.name) for x in f.children), Tuple{(juliatype(y) for y in f.children)...}, } struct Map <: Type name::String keysSorted::Base.Bool end Type(::Base.Type{Dict{K,V}}) where {K,V} = Map("map", false) children(::Base.Type{Dict{K,V}}) where {K,V} = [Field("entries", Arrow.KeyValue{K,V}, nothing)] StructTypes.StructType(::Base.Type{Map}) = StructTypes.Struct() juliatype(f, x::Map) = Dict{juliatype(f.children[1].children[1]),juliatype(f.children[1].children[2])} Type(::Base.Type{Arrow.KeyValue{K,V}}) where {K,V} = Struct("struct") children(::Base.Type{Arrow.KeyValue{K,V}}) where {K,V} = [Field("key", K, nothing), Field("value", V, nothing)] struct Null <: Type name::String end Type(::Base.Type{Missing}) = Null("null") StructTypes.StructType(::Base.Type{Null}) = StructTypes.Struct() juliatype(f, x::Null) = Missing struct Utf8 <: Type name::String end Type(::Base.Type{<:String}) = Utf8("utf8") StructTypes.StructType(::Base.Type{Utf8}) = StructTypes.Struct() juliatype(f, x::Utf8) = String struct LargeUtf8 <: Type name::String end StructTypes.StructType(::Base.Type{LargeUtf8}) = StructTypes.Struct() juliatype(f, x::LargeUtf8) = String struct Binary <: Type name::String end Type(::Base.Type{Vector{UInt8}}) = Binary("binary") children(::Base.Type{Vector{UInt8}}) = Field[] StructTypes.StructType(::Base.Type{Binary}) = StructTypes.Struct() juliatype(f, x::Binary) = Vector{UInt8} struct LargeBinary <: Type name::String end StructTypes.StructType(::Base.Type{LargeBinary}) = StructTypes.Struct() juliatype(f, x::LargeBinary) = Vector{UInt8} struct Bool <: Type name::String end Type(::Base.Type{Base.Bool}) = Bool("bool") StructTypes.StructType(::Base.Type{Bool}) = StructTypes.Struct() juliatype(f, x::Bool) = Base.Bool StructTypes.subtypekey(::Base.Type{Type}) = :name const SUBTYPES = @eval ( int=Int, floatingpoint=FloatingPoint, fixedsizebinary=FixedSizeBinary, decimal=Decimal, timestamp=Timestamp, duration=Duration, date=Date, time=Time, interval=Interval, union=UnionT, list=List, largelist=LargeList, fixedsizelist=FixedSizeList, ($(Symbol("struct")))=Struct, map=Map, null=Null, utf8=Utf8, largeutf8=LargeUtf8, binary=Binary, largebinary=LargeBinary, bool=Bool, ) StructTypes.subtypes(::Base.Type{Type}) = SUBTYPES const Metadata = Union{Nothing,Vector{NamedTuple{(:key, :value),Tuple{String,String}}}} Metadata() = nothing mutable struct DictEncoding id::Int64 indexType::Type isOrdered::Base.Bool end DictEncoding() = DictEncoding(0, Type(), false) StructTypes.StructType(::Base.Type{DictEncoding}) = StructTypes.Mutable() mutable struct Field name::String nullable::Base.Bool type::Type children::Vector{Field} dictionary::Union{DictEncoding,Nothing} metadata::Metadata end Field() = Field("", true, Type(), Field[], nothing, Metadata()) StructTypes.StructType(::Base.Type{Field}) = StructTypes.Mutable() Base.copy(f::Field) = Field(f.name, f.nullable, f.type, f.children, f.dictionary, f.metadata) function juliatype(f::Field) T = juliatype(f, f.type) return f.nullable ? Union{T,Missing} : T end function Field(nm, ::Base.Type{T}, dictencodings) where {T} S = Arrow.maybemissing(T) type = Type(S) ch = children(S) if dictencodings !== nothing && haskey(dictencodings, nm) dict = dictencodings[nm] else dict = nothing end return Field(nm, T !== S, type, ch, dict, nothing) end mutable struct Schema fields::Vector{Field} metadata::Metadata end Schema() = Schema(Field[], Metadata()) StructTypes.StructType(::Base.Type{Schema}) = StructTypes.Mutable() struct Offsets{T} <: AbstractVector{T} data::Vector{T} end Base.size(x::Offsets) = size(x.data) Base.getindex(x::Offsets, i::Base.Int) = getindex(x.data, i) mutable struct FieldData name::String count::Int64 VALIDITY::Union{Nothing,Vector{Int8}} OFFSET::Union{Nothing,Offsets} TYPE_ID::Union{Nothing,Vector{Int8}} DATA::Union{Nothing,Vector{Any}} children::Vector{FieldData} end FieldData() = FieldData("", 0, nothing, nothing, nothing, nothing, FieldData[]) StructTypes.StructType(::Base.Type{FieldData}) = StructTypes.Mutable() function FieldData(nm, ::Base.Type{T}, col, dictencodings) where {T} if dictencodings !== nothing && haskey(dictencodings, nm) refvals = DataAPI.refarray(col.data) if refvals !== col.data IT = eltype(refvals) col = (x - one(T) for x in refvals) else _, de = dictencodings[nm] IT = de.indexType vals = unique(col) col = Arrow.DictEncoder(col, vals, Arrow.encodingtype(length(vals))) end return FieldData(nm, IT, col, nothing) end S = Arrow.maybemissing(T) len = Arrow._length(col) VALIDITY = OFFSET = TYPE_ID = DATA = nothing children = FieldData[] if S <: Pair return FieldData( nm, Vector{Arrow.KeyValue{Arrow._keytype(S),Arrow._valtype(S)}}, (Arrow.KeyValue(k, v) for (k, v) in pairs(col)), ) elseif S !== Missing # VALIDITY VALIDITY = Int8[!ismissing(x) for x in col] # OFFSET if S <: Vector || S == String lenfun = S == String ? x -> ismissing(x) ? 0 : sizeof(x) : x -> ismissing(x) ? 0 : length(x) tot = sum(lenfun, col) if tot > 2147483647 OFFSET = String[String(lenfun(x)) for x in col] pushfirst!(OFFSET, "0") else OFFSET = Int32[ismissing(x) ? 0 : lenfun(x) for x in col] pushfirst!(OFFSET, 0) end OFFSET = Offsets(OFFSET) push!( children, FieldData( "item", eltype(S), Arrow.flatten(skipmissing(col)), dictencodings, ), ) elseif S <: NTuple if Arrow.ArrowTypes.gettype(Arrow.ArrowTypes.ArrowKind(S)) == UInt8 DATA = [ ismissing(x) ? Arrow.ArrowTypes.default(S) : String(collect(x)) for x in col ] else push!( children, FieldData( "item", Arrow.ArrowTypes.gettype(Arrow.ArrowTypes.ArrowKind(S)), Arrow.flatten( coalesce(x, Arrow.ArrowTypes.default(S)) for x in col ), dictencodings, ), ) end elseif S <: NamedTuple for (nm, typ) in zip(fieldnames(S), fieldtypes(S)) push!( children, FieldData( String(nm), typ, (getfield(x, nm) for x in col), dictencodings, ), ) end elseif S <: Arrow.UnionT U = eltype(S) tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S) TYPE_ID = [x === missing ? 0 : tids[Arrow.isatypeid(x, U)] for x in col] if Arrow.unionmode(S) == Arrow.Meta.UnionMode.Dense offs = zeros(Int32, fieldcount(U)) OFFSET = Int32[] for x in col idx = x === missing ? 1 : Arrow.isatypeid(x, U) push!(OFFSET, offs[idx]) offs[idx] += 1 end for i = 1:fieldcount(U) SS = fieldtype(U, i) push!( children, FieldData( "$i", SS, Arrow.filtered( i == 1 ? Union{SS,Missing} : Arrow.maybemissing(SS), col, ), dictencodings, ), ) end else for i = 1:fieldcount(U) SS = fieldtype(U, i) push!( children, FieldData("$i", SS, Arrow.replaced(SS, col), dictencodings), ) end end elseif S <: KeyValue push!( children, FieldData("key", Arrow.keyvalueK(S), (x.key for x in col), dictencodings), ) push!( children, FieldData( "value", Arrow.keyvalueV(S), (x.value for x in col), dictencodings, ), ) end end return FieldData(nm, len, VALIDITY, OFFSET, TYPE_ID, DATA, children) end mutable struct RecordBatch count::Int64 columns::Vector{FieldData} end RecordBatch() = RecordBatch(0, FieldData[]) StructTypes.StructType(::Base.Type{RecordBatch}) = StructTypes.Mutable() mutable struct DictionaryBatch id::Int64 data::RecordBatch end DictionaryBatch() = DictionaryBatch(0, RecordBatch()) StructTypes.StructType(::Base.Type{DictionaryBatch}) = StructTypes.Mutable() mutable struct DataFile <: Tables.AbstractColumns schema::Schema batches::Vector{RecordBatch} dictionaries::Vector{DictionaryBatch} end Base.propertynames(x::DataFile) = (:schema, :batches, :dictionaries) function Base.getproperty(df::DataFile, nm::Symbol) if nm === :schema return getfield(df, :schema) elseif nm === :batches return getfield(df, :batches) elseif nm === :dictionaries return getfield(df, :dictionaries) end return Tables.getcolumn(df, nm) end DataFile() = DataFile(Schema(), RecordBatch[], DictionaryBatch[]) StructTypes.StructType(::Base.Type{DataFile}) = StructTypes.Mutable() parsefile(file) = JSON3.read(Mmap.mmap(file), DataFile) # make DataFile satisfy Tables.jl interface function Tables.partitions(x::DataFile) if isempty(x.batches) # special case empty batches by producing a single DataFile w/ schema return (DataFile(x.schema, RecordBatch[], x.dictionaries),) else return ( DataFile(x.schema, [x.batches[i]], x.dictionaries) for i = 1:length(x.batches) ) end end Tables.columns(x::DataFile) = x function Tables.schema(x::DataFile) names = map(x -> x.name, x.schema.fields) types = map(x -> juliatype(x), x.schema.fields) return Tables.Schema(names, types) end Tables.columnnames(x::DataFile) = map(x -> Symbol(x.name), x.schema.fields) function Tables.getcolumn(x::DataFile, i::Base.Int) field = x.schema.fields[i] type = juliatype(field) return ChainedVector( ArrowArray{type}[ ArrowArray{type}( field, length(x.batches) > 0 ? x.batches[j].columns[i] : FieldData(), x.dictionaries, ) for j = 1:length(x.batches) ], ) end function Tables.getcolumn(x::DataFile, nm::Symbol) i = findfirst(x -> x.name == String(nm), x.schema.fields) return Tables.getcolumn(x, i) end struct ArrowArray{T} <: AbstractVector{T} field::Field fielddata::FieldData dictionaries::Vector{DictionaryBatch} end ArrowArray(f::Field, fd::FieldData, d) = ArrowArray{juliatype(f)}(f, fd, d) Base.size(x::ArrowArray) = (x.fielddata.count,) function Base.getindex(x::ArrowArray{T}, i::Base.Int) where {T} @boundscheck checkbounds(x, i) S = Base.nonmissingtype(T) if x.field.dictionary !== nothing fielddata = x.dictionaries[findfirst(y -> y.id == x.field.dictionary.id, x.dictionaries)].data.columns[1] field = copy(x.field) field.dictionary = nothing idx = x.fielddata.DATA[i] + 1 return ArrowArray(field, fielddata, x.dictionaries)[idx] end if T === Missing return missing elseif S <: UnionT U = eltype(S) tids = Arrow.typeids(S) === nothing ? (0:fieldcount(U)) : Arrow.typeids(S) typeid = tids[x.fielddata.TYPE_ID[i]] if Arrow.unionmode(S) == Arrow.Meta.UnionMode.DENSE off = x.fielddata.OFFSET[i] return ArrowArray( x.field.children[typeid + 1], x.fielddata.children[typeid + 1], x.dictionaries, )[off] else return ArrowArray( x.field.children[typeid + 1], x.fielddata.children[typeid + 1], x.dictionaries, )[i] end end x.fielddata.VALIDITY[i] == 0 && return missing if S <: Vector{UInt8} return copy(codeunits(x.fielddata.DATA[i])) elseif S <: String return x.fielddata.DATA[i] elseif S <: Vector offs = x.fielddata.OFFSET A = ArrowArray{eltype(S)}( x.field.children[1], x.fielddata.children[1], x.dictionaries, ) return A[(offs[i] + 1):offs[i + 1]] elseif S <: Dict offs = x.fielddata.OFFSET A = ArrowArray(x.field.children[1], x.fielddata.children[1], x.dictionaries) return Dict(y.key => y.value for y in A[(offs[i] + 1):offs[i + 1]]) elseif S <: Tuple if Arrow.ArrowTypes.gettype(Arrow.ArrowTypes.ArrowKind(S)) == UInt8 A = x.fielddata.DATA return Tuple(map(UInt8, collect(A[i][1:(x.field.type.byteWidth)]))) else sz = x.field.type.listSize A = ArrowArray{Arrow.ArrowTypes.gettype(Arrow.ArrowTypes.ArrowKind(S))}( x.field.children[1], x.fielddata.children[1], x.dictionaries, ) off = (i - 1) * sz + 1 return Tuple(A[off:(off + sz - 1)]) end elseif S <: NamedTuple data = ( ArrowArray(x.field.children[j], x.fielddata.children[j], x.dictionaries)[i] for j = 1:length(x.field.children) ) return NamedTuple{fieldnames(S)}(Tuple(data)) elseif S == Int64 || S == UInt64 return parse(S, x.fielddata.DATA[i]) elseif S <: Arrow.Decimal str = x.fielddata.DATA[i] return S(parse(Int128, str)) elseif S <: Arrow.Date || S <: Arrow.Time val = x.fielddata.DATA[i] return Arrow.storagetype(S) == Int32 ? S(val) : S(parse(Int64, val)) elseif S <: Arrow.Timestamp return S(parse(Int64, x.fielddata.DATA[i])) else return S(x.fielddata.DATA[i]) end end # take any Tables.jl source and write out arrow json datafile function DataFile(source) fields = Field[] metadata = nothing # TODO? batches = RecordBatch[] dictionaries = DictionaryBatch[] dictencodings = Dict{String,Tuple{Base.Type,DictEncoding}}() dictid = Ref(0) for (i, tbl1) in Tables.partitions(source) tbl = Arrow.toarrowtable(Table.Columns(tbl1)) if i == 1 sch = Tables.schema(tbl) for (nm, T, col) in zip(sch.names, sch.types, tbl) if col isa Arrow.DictEncode id = dictid[] dictid[] += 1 codes = DataAPI.refarray(col.data) if codes !== col.data IT = Type(eltype(codes)) else IT = Type(Arrow.encodingtype(length(unique(col)))) end dictencodings[String(nm)] = (T, DictEncoding(id, IT, false)) end push!(fields, Field(String(nm), T, dictencodings)) end end # build record batch len = Tables.rowcount(tbl) columns = FieldData[] for (nm, T, col) in zip(sch.names, sch.types, tbl) push!(columns, FieldData(String(nm), T, col, dictencodings)) end push!(batches, RecordBatch(len, columns)) # build dictionaries for (nm, (T, dictencoding)) in dictencodings column = FieldData(nm, T, Tables.getcolumn(tbl, nm), nothing) recordbatch = RecordBatch(len, [column]) push!(dictionaries, DictionaryBatch(dictencoding.id, recordbatch)) end end schema = Schema(fields, metadata) return DataFile(schema, batches, dictionaries) end function Base.isequal(df::DataFile, tbl::Arrow.Table) Arrow.is_equivalent_schema(Tables.schema(df), Tables.schema(tbl)) || return false i = 1 for (col1, col2) in zip(Tables.Columns(df), Tables.Columns(tbl)) if !isequal(col1, col2) @show i return false end i += 1 end return true end end ================================================ FILE: test/dates.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import Dates import TimeZones struct WrappedDate x::Dates.Date end ArrowTypes.arrowname(::Type{WrappedDate}) = Symbol("JuliaLang.WrappedDate") ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.WrappedDate")}) = WrappedDate struct WrappedTime x::Dates.Time end ArrowTypes.arrowname(::Type{WrappedTime}) = Symbol("JuliaLang.WrappedTime") ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.WrappedTime")}) = WrappedTime struct WrappedDateTime x::Dates.DateTime end ArrowTypes.arrowname(::Type{WrappedDateTime}) = Symbol("JuliaLang.WrappedDateTime") ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.WrappedDateTime")}) = WrappedDateTime struct WrappedZonedDateTime x::TimeZones.ZonedDateTime end ArrowTypes.arrowname(::Type{WrappedZonedDateTime}) = Symbol("JuliaLang.WrappedZonedDateTime") ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.WrappedZonedDateTime")}) = WrappedZonedDateTime @testset "Date and time wrappers with missing" begin for T in (WrappedDate, WrappedTime, WrappedDateTime, WrappedZonedDateTime) if T == WrappedZonedDateTime time = T(Dates.now(TimeZones.tz"UTC")) else time = T(Dates.now()) end table = (; x=[missing, missing, time, missing, time]) io = Arrow.tobuffer(table) tbl = Arrow.Table(io) @test isequal(collect(tbl.x), table.x) end end @testset "`default(T) isa T`" begin for T in ( Dates.Date, Dates.Time, Dates.DateTime, TimeZones.ZonedDateTime, Dates.Nanosecond, Dates.Millisecond, Dates.Second, Dates.Day, Dates.Month, Dates.Year, ) @test Arrow.ArrowTypes.default(T) isa T end end ================================================ FILE: test/integrationtest.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. using Arrow, Tables, Test include(joinpath(dirname(pathof(Arrow)), "../test/arrowjson.jl")) # using .ArrowJSON function runcommand(jsonname, arrowname, mode, verbose) if jsonname == "" error("must provide json file name") end if arrowname == "" error("must provide arrow file name") end if mode == "ARROW_TO_JSON" tbl = Arrow.Table(arrowname) df = ArrowJSON.DataFile(tbl) open(jsonname, "w") do io JSON3.write(io, df) end elseif mode == "JSON_TO_ARROW" df = ArrowJSON.parsefile(jsonname) open(arrowname, "w") do io Arrow.write(io, df) end elseif mode == "VALIDATE" df = ArrowJSON.parsefile(jsonname) tbl = Arrow.Table(arrowname) @test isequal(df, tbl) else error("unknown integration test mode: $mode") end return end ================================================ FILE: test/pyarrow_roundtrip.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ENV["PYTHON"] = "python3" import PyCall pa = PyCall.pyimport("pyarrow") include(joinpath(dirname(pathof(Arrow)), "../test/testtables.jl")) for (nm, t, writekw, readkw, extratests) in testtables nm == "unions" && continue @testset "pyarrow roundtrip: $nm" begin io = IOBuffer() Arrow.write(io, t; writekw...) seekstart(io) buf = PyCall.pybytes(take!(io)) reader = pa.ipc.open_stream(buf) sink = pa.BufferOutputStream() writer = pa.ipc.new_stream(sink, reader.schema) for batch in reader writer.write_batch(batch) end writer.close() buf = sink.getvalue() jbytes = copy(reinterpret(UInt8, buf)) tt = Arrow.Table(jbytes) end @testset "pyarrow roundtrip w/ compression: $nm" begin io = IOBuffer() Arrow.write(io, t; compress=((:lz4, :zstd)[rand(1:2)]), writekw...) seekstart(io) buf = PyCall.pybytes(take!(io)) reader = pa.ipc.open_stream(buf) sink = pa.BufferOutputStream() writer = pa.ipc.new_stream(sink, reader.schema) for batch in reader writer.write_batch(batch) end writer.close() buf = sink.getvalue() jbytes = copy(reinterpret(UInt8, buf)) tt = Arrow.Table(jbytes) end end f1 = pa.field("f1", pa.float64(), true) f2 = pa.field("f2", pa.int64(), false) fu = pa.field("col1", pa.union([f1, f2], "dense")) sch = pa.schema([fu]) xs = pa.array([2.0, 4.0, PyCall.pynothing[]], type=pa.float64()) ys = pa.array([1, 3], type=pa.int64()) types = pa.array([0, 1, 0, 1, 1], type=pa.int8()) offsets = pa.array([0, 0, 1, 1, 2], type=pa.int32()) union_arr = pa.UnionArray.from_dense(types, offsets, [xs, ys]) data = [union_arr] batch = pa.record_batch(data, names=["col1"]) sink = pa.BufferOutputStream() writer = pa.ipc.new_stream(sink, batch.schema) writer.write_batch(batch) writer.close() buf = sink.getvalue() jbytes = copy(reinterpret(UInt8, buf)) tt = Arrow.Table(jbytes) ================================================ FILE: test/runtests.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. using Test using Arrow using ArrowTypes using Tables using Dates using PooledArrays using TimeZones using UUIDs using Sockets using CategoricalArrays using DataAPI using FilePathsBase using DataFrames using OffsetArrays import Random: randstring using TestSetExtensions: ExtendedTestSet # this formulation tests the loaded ArrowTypes, even if it's not the dev version # within the mono-repo include(joinpath(dirname(pathof(ArrowTypes)), "../test/tests.jl")) include(joinpath(@__DIR__, "testtables.jl")) include(joinpath(@__DIR__, "testappend.jl")) include(joinpath(@__DIR__, "integrationtest.jl")) include(joinpath(@__DIR__, "dates.jl")) struct CustomStruct x::Int y::Float64 z::String end struct CustomStruct2{sym} x::Int end @testset ExtendedTestSet "Arrow" begin @testset "table roundtrips" begin for case in testtables testtable(case...) end end # @testset "table roundtrips" @testset "table append" begin # skip windows since file locking prevents mktemp cleanup if !Sys.iswindows() for case in testtables testappend(case...) end testappend_partitions() for compression_option in (:lz4, :zstd) testappend_compression(compression_option) end end end # @testset "table append" @testset "arrow json integration tests" begin for file in readdir(joinpath(dirname(pathof(Arrow)), "../test/arrowjson")) jsonfile = joinpath(joinpath(dirname(pathof(Arrow)), "../test/arrowjson"), file) @testset "integration test for $jsonfile" begin df = ArrowJSON.parsefile(jsonfile) io = Arrow.tobuffer(df) tbl = Arrow.Table(io; convert=false) @test isequal(df, tbl) end end end # @testset "arrow json integration tests" @testset "abstract path" begin # Make a custom path type that simulates how AWSS3.jl's S3Path works struct CustomPath <: AbstractPath path::PosixPath end Base.read(p::CustomPath) = read(p.path) io = Arrow.tobuffer((col=[0],)) tt = Arrow.Table(io) mktempdir() do dir p = Path(joinpath(dir, "test.arrow")) Arrow.write(p, tt) @test isfile(p) # skip windows since file locking prevents mktemp cleanup if !Sys.iswindows() tt2 = Arrow.Table(p) @test values(tt) == values(tt2) tt3 = Arrow.Table(CustomPath(p)) @test values(tt) == values(tt3) end end end # @testset "abstract path" @testset "misc" begin @testset "# multiple record batches" begin t = Tables.partitioner(( (col1=Union{Int64,Missing}[1, 2, 3, 4, 5, 6, 7, 8, 9, missing],), (col1=Union{Int64,Missing}[missing, 11],), )) io = Arrow.tobuffer(t) tt = Arrow.Table(io) @test length(tt) == 1 @test isequal( tt.col1, vcat([1, 2, 3, 4, 5, 6, 7, 8, 9, missing], [missing, 11]), ) @test eltype(tt.col1) === Union{Int64,Missing} # Arrow.Stream seekstart(io) str = Arrow.Stream(io) @test eltype(str) == Arrow.Table @test !Base.isdone(str) state = iterate(str) @test state !== nothing tt, st = state @test length(tt) == 1 @test isequal(tt.col1, [1, 2, 3, 4, 5, 6, 7, 8, 9, missing]) state = iterate(str, st) @test state !== nothing tt, st = state @test length(tt) == 1 @test isequal(tt.col1, [missing, 11]) @test iterate(str, st) === nothing @test isequal(collect(str)[1].col1, [1, 2, 3, 4, 5, 6, 7, 8, 9, missing]) @test isequal(collect(str)[2].col1, [missing, 11]) end @testset "# dictionary batch isDelta" begin t = ( col1=Int64[1, 2, 3, 4], col2=Union{String,Missing}["hey", "there", "sailor", missing], col3=NamedTuple{ (:a, :b), Tuple{Int64,Union{Missing,NamedTuple{(:c,),Tuple{String}}}}, }[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",)), ], ) t2 = ( col1=Int64[1, 2, 5, 6], col2=Union{String,Missing}["hey", "there", "sailor2", missing], col3=NamedTuple{ (:a, :b), Tuple{Int64,Union{Missing,NamedTuple{(:c,),Tuple{String}}}}, }[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(5), b=(c="sailor2",)), (a=Int64(4), b=(c="jo-bob",)), ], ) tt = Tables.partitioner((t, t2)) tt = Arrow.Table(Arrow.tobuffer(tt; dictencode=true, dictencodenested=true)) @test tt.col1 == [1, 2, 3, 4, 1, 2, 5, 6] @test isequal( tt.col2, ["hey", "there", "sailor", missing, "hey", "there", "sailor2", missing], ) @test isequal( tt.col3, vcat( NamedTuple{ (:a, :b), Tuple{Int64,Union{Missing,NamedTuple{(:c,),Tuple{String}}}}, }[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",)), ], NamedTuple{ (:a, :b), Tuple{Int64,Union{Missing,NamedTuple{(:c,),Tuple{String}}}}, }[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(5), b=(c="sailor2",)), (a=Int64(4), b=(c="jo-bob",)), ], ), ) end @testset "metadata" begin t = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) meta = Dict("key1" => "value1", "key2" => "value2") meta2 = Dict("colkey1" => "colvalue1", "colkey2" => "colvalue2") tt = Arrow.Table( Arrow.tobuffer(t; colmetadata=Dict(:col1 => meta2), metadata=meta), ) @test length(tt) == length(t) @test tt.col1 == t.col1 @test eltype(tt.col1) === Int64 @test Arrow.getmetadata(tt) == Arrow.toidict(meta) @test Arrow.getmetadata(tt.col1) == Arrow.toidict(meta2) t = (col1=collect(1:10), col2=collect('a':'j'), col3=collect(1:10)) meta = ("key1" => :value1, :key2 => "value2") meta2 = ("colkey1" => :colvalue1, :colkey2 => "colvalue2") meta3 = ("colkey3" => :colvalue3,) tt = Arrow.Table( Arrow.tobuffer( t; colmetadata=Dict(:col2 => meta2, :col3 => meta3), metadata=meta, ), ) @test Arrow.getmetadata(tt) == Arrow.toidict(String(k) => String(v) for (k, v) in meta) @test Arrow.getmetadata(tt.col1) === nothing @test Arrow.getmetadata(tt.col2)["colkey1"] == "colvalue1" @test Arrow.getmetadata(tt.col2)["colkey2"] == "colvalue2" @test Arrow.getmetadata(tt.col3)["colkey3"] == "colvalue3" end @testset "# custom compressors" begin lz4 = Arrow.CodecLz4.LZ4FrameCompressor(; compressionlevel=8) Arrow.CodecLz4.TranscodingStreams.initialize(lz4) t = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) tt = Arrow.Table(Arrow.tobuffer(t; compress=lz4)) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) zstd = Arrow.CodecZstd.ZstdCompressor(; level=8) Arrow.CodecZstd.TranscodingStreams.initialize(zstd) t = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) tt = Arrow.Table(Arrow.tobuffer(t; compress=zstd)) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) end @testset "# custom alignment" begin t = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) tt = Arrow.Table(Arrow.tobuffer(t; alignment=64)) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) end @testset "# 53" begin s = "a"^100 t = (a=[SubString(s, 1:10), SubString(s, 11:20)],) tt = Arrow.Table(Arrow.tobuffer(t)) @test tt.a == ["aaaaaaaaaa", "aaaaaaaaaa"] end @testset "# 49" begin @test_throws SystemError Arrow.Table("file_that_doesnt_exist") @test_throws SystemError Arrow.Table(p"file_that_doesnt_exist") end @testset "# 52" begin t = (a=Arrow.DictEncode(string.(1:129)),) tt = Arrow.Table(Arrow.tobuffer(t)) end @testset "# 60: unequal column lengths" begin io = IOBuffer() @test_throws ArgumentError Arrow.write( io, (a=Int[], b=["asd"], c=collect(1:100)), ) end @testset "# nullability of custom extension types" begin t = (a=['a', missing],) tt = Arrow.Table(Arrow.tobuffer(t)) @test isequal(tt.a, ['a', missing]) end @testset "# automatic custom struct serialization/deserialization" begin t = (col1=[CustomStruct(1, 2.3, "hey"), CustomStruct(4, 5.6, "there")],) Arrow.ArrowTypes.arrowname(::Type{CustomStruct}) = Symbol("JuliaLang.CustomStruct") Arrow.ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.CustomStruct")}, S) = CustomStruct tt = Arrow.Table(Arrow.tobuffer(t)) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) end @testset "# 76" begin t = (col1=NamedTuple{(:a,),Tuple{Union{Int,String}}}[(a=1,), (a="x",)],) tt = Arrow.Table(Arrow.tobuffer(t)) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) end @testset "# 89 etc. - UUID FixedSizeListKind overloads" begin @test Arrow.ArrowTypes.gettype(Arrow.ArrowTypes.ArrowKind(UUID)) == UInt8 @test Arrow.ArrowTypes.getsize(Arrow.ArrowTypes.ArrowKind(UUID)) == 16 end @testset "# 98" begin t = ( a=[Nanosecond(0), Nanosecond(1)], b=[uuid4(), uuid4()], c=[missing, Nanosecond(1)], ) tt = Arrow.Table(Arrow.tobuffer(t)) @test copy(tt.a) isa Vector{Nanosecond} @test copy(tt.b) isa Vector{UUID} @test copy(tt.c) isa Vector{Union{Missing,Nanosecond}} end @testset "# copy on DictEncoding w/ missing values" begin x = PooledArray(["hey", missing]) x2 = Arrow.toarrowvector(x) @test isequal(copy(x2), x) end @testset "# some dict encoding coverage" begin # signed indices for DictEncodedKind #112 #113 #114 av = Arrow.toarrowvector(PooledArray(repeat(["a", "b"], inner=5))) @test isa(first(av.indices), Signed) av = Arrow.toarrowvector(CategoricalArray(repeat(["a", "b"], inner=5))) @test isa(first(av.indices), Signed) av = Arrow.toarrowvector(CategoricalArray(["a", "bb", missing])) @test isa(first(av.indices), Signed) @test length(av) == 3 @test eltype(av) == Union{String,Missing} av = Arrow.toarrowvector(CategoricalArray(["a", "bb", "ccc"])) @test isa(first(av.indices), Signed) @test length(av) == 3 @test eltype(av) == String end @testset "# 120" begin x = PooledArray(["hey", missing]) x2 = Arrow.toarrowvector(x) @test eltype(DataAPI.refpool(x2)) == Union{Missing,String} @test eltype(DataAPI.levels(x2)) == String @test DataAPI.refarray(x2) == [1, 2] end @testset "# 121" begin a = PooledArray(repeat(string.('S', 1:130), inner=5), compress=true) @test eltype(a.refs) == UInt8 av = Arrow.toarrowvector(a) @test eltype(av.indices) == Int16 end @testset "# 123" begin t = (x=collect(zip(rand(10), rand(10))),) tt = Arrow.Table(Arrow.tobuffer(t)) @test tt.x == t.x end @testset "# 144" begin t = Tables.partitioner(( (a=Arrow.DictEncode([1, 2, 3]),), (a=Arrow.DictEncode(fill(1, 129)),), )) tt = Arrow.Table(Arrow.tobuffer(t)) @test length(tt.a) == 132 end @testset "# 126" begin # XXX This test also captures a race condition in multithreaded # writes of dictionary encoded arrays t = Tables.partitioner(( (a=Arrow.toarrowvector(PooledArray([1, 2, 3])),), (a=Arrow.toarrowvector(PooledArray([1, 2, 3, 4])),), (a=Arrow.toarrowvector(PooledArray([1, 2, 3, 4, 5])),), )) tt = Arrow.Table(Arrow.tobuffer(t)) @test length(tt.a) == 12 @test tt.a == [1, 2, 3, 1, 2, 3, 4, 1, 2, 3, 4, 5] t = Tables.partitioner(( ( a=Arrow.toarrowvector( PooledArray([1, 2, 3], signed=true, compress=true), ), ), (a=Arrow.toarrowvector(PooledArray(collect(1:129))),), )) io = IOBuffer() @test_logs (:error, "error writing arrow data on partition = 2") begin @test_throws ErrorException Arrow.write(io, t) end end @testset "# 75" begin tbl = Arrow.Table(Arrow.tobuffer((sets=[Set([1, 2, 3]), Set([1, 2, 3])],))) @test eltype(tbl.sets) <: Set end @testset "# 85" begin tbl = Arrow.Table(Arrow.tobuffer((tups=[(1, 3.14, "hey"), (1, 3.14, "hey")],))) @test eltype(tbl.tups) <: Tuple end @testset "Nothing" begin tbl = Arrow.Table(Arrow.tobuffer((nothings=[nothing, nothing, nothing],))) @test tbl.nothings == [nothing, nothing, nothing] end @testset "arrowmetadata" begin # arrowmetadata t = (col1=[CustomStruct2{:hey}(1), CustomStruct2{:hey}(2)],) ArrowTypes.arrowname(::Type{<:CustomStruct2}) = Symbol("CustomStruct2") @test_logs (:warn, r"unsupported ARROW:extension:name type: \"CustomStruct2\"") begin tbl = Arrow.Table(Arrow.tobuffer(t)) end @test eltype(tbl.col1) <: NamedTuple ArrowTypes.arrowmetadata(::Type{CustomStruct2{sym}}) where {sym} = sym ArrowTypes.JuliaType(::Val{:CustomStruct2}, S, meta) = CustomStruct2{Symbol(meta)} tbl = Arrow.Table(Arrow.tobuffer(t)) @test eltype(tbl.col1) == CustomStruct2{:hey} end @testset "# 166" begin t = (col1=[zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.NANOSECOND,nothing})],) tbl = Arrow.Table(Arrow.tobuffer(t)) @test_logs ( :warn, r"automatically converting Arrow.Timestamp with precision = NANOSECOND", ) begin @test tbl.col1[1] == Dates.DateTime(1970) end end @testset "# 95; Arrow.ToTimestamp" begin x = [ZonedDateTime(Dates.DateTime(2020), tz"Europe/Paris")] c = Arrow.ToTimestamp(x) @test eltype(c) == Arrow.Timestamp{Arrow.Flatbuf.TimeUnit.MILLISECOND,Symbol("Europe/Paris")} @test c[1] == Arrow.Timestamp{Arrow.Flatbuf.TimeUnit.MILLISECOND,Symbol("Europe/Paris")}( 1577833200000, ) end @testset "# 158" begin # arrow ipc stream generated from pyarrow with no record batches bytes = UInt8[ 0xff, 0xff, 0xff, 0xff, 0x78, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x0c, 0x00, 0x06, 0x00, 0x05, 0x00, 0x08, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x14, 0x00, 0x08, 0x00, 0x06, 0x00, 0x07, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x10, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x07, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, ] tbl = Arrow.Table(bytes) @test length(tbl.a) == 0 @test eltype(tbl.a) == Union{Int64,Missing} end @testset "# 181" begin # XXX this test hangs on Julia 1.12 when using a deeper nesting d = Dict{Int,Int}() for i = 1:1 d = Dict(i => d) end tbl = (x=[d],) msg = "reached nested serialization level (2) deeper than provided max depth argument (1); to increase allowed nesting level, pass `maxdepth=X`" @test_throws ErrorException(msg) Arrow.tobuffer(tbl; maxdepth=1) @test Arrow.Table(Arrow.tobuffer(tbl; maxdepth=5)).x == tbl.x end @testset "# 167" begin t = (col1=[["boop", "she"], ["boop", "she"], ["boo"]],) tbl = Arrow.Table(Arrow.tobuffer(t)) @test eltype(tbl.col1) <: AbstractVector{String} end @testset "# 200 VersionNumber" begin t = (col1=[v"1"],) tbl = Arrow.Table(Arrow.tobuffer(t)) @test eltype(tbl.col1) == VersionNumber end @testset "`show`" begin str = nothing table = (; a=1:5, b=fill(1.0, 5)) arrow_table = Arrow.Table(Arrow.tobuffer(table)) # 2 and 3-arg show with no metadata for outer str in (sprint(show, arrow_table), sprint(show, MIME"text/plain"(), arrow_table)) @test length(str) < 100 @test occursin("5 rows", str) @test occursin("2 columns", str) @test occursin("Int", str) @test occursin("Float64", str) @test !occursin("metadata entries", str) end # 2-arg show with metadata big_dict = Dict((randstring(rand(5:10)) => randstring(rand(1:3)) for _ = 1:100)) arrow_table = Arrow.Table(Arrow.tobuffer(table; metadata=big_dict)) str2 = sprint(show, arrow_table) @test length(str2) > length(str) @test length(str2) < 200 @test occursin("metadata entries", str2) # 3-arg show with metadata str3 = sprint( show, MIME"text/plain"(), arrow_table; context=IOContext(IOBuffer(), :displaysize => (24, 100), :limit => true), ) @test length(str3) < 1000 # some but not too many `=>`'s for printing the metadata @test 5 < length(collect(eachmatch(r"=>", str3))) < 20 end @testset "# 194" begin @test isempty(Arrow.Table(Arrow.tobuffer(Dict{Symbol,Vector}()))) end @testset "# 229" begin struct Foo229{x} y::String z::Int end Arrow.ArrowTypes.arrowname(::Type{<:Foo229}) = Symbol("JuliaLang.Foo229") Arrow.ArrowTypes.ArrowType(::Type{Foo229{x}}) where {x} = Tuple{String,String,Int} Arrow.ArrowTypes.toarrow(row::Foo229{x}) where {x} = (String(x), row.y, row.z) Arrow.ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.Foo229")}, ::Any) = Foo229 Arrow.ArrowTypes.fromarrow(::Type{<:Foo229}, x, y, z) = Foo229{Symbol(x)}(y, z) cols = ( k1=[Foo229{:a}("a", 1), Foo229{:b}("b", 2)], k2=[Foo229{:c}("c", 3), Foo229{:d}("d", 4)], ) tbl = Arrow.Table(Arrow.tobuffer(cols)) @test tbl.k1 == cols.k1 @test tbl.k2 == cols.k2 end @testset "# PR 234" begin # bugfix parsing primitive arrays buf = [ 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x14, 0x00, 0x00, 0x00, 0x10, 0x00, 0x0c, 0x00, 0x08, 0x00, 0x04, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] struct TestData <: Arrow.FlatBuffers.Table bytes::Vector{UInt8} pos::Base.Int end function Base.getproperty(x::TestData, field::Symbol) if field === :DataInt32 o = Arrow.FlatBuffers.offset(x, 12) o != 0 && return Arrow.FlatBuffers.Array{Int32}(x, o) else @warn "field $field not supported" end end d = Arrow.FlatBuffers.getrootas(TestData, buf, 0) @test d.DataInt32 == UInt32[1, 2, 3] end @testset "# test multiple inputs treated as one table" begin t = (col1=[1, 2, 3, 4, 5], col2=[1.2, 2.3, 3.4, 4.5, 5.6]) tbl = Arrow.Table([Arrow.tobuffer(t), Arrow.tobuffer(t)]) @test tbl.col1 == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] @test tbl.col2 == [1.2, 2.3, 3.4, 4.5, 5.6, 1.2, 2.3, 3.4, 4.5, 5.6] # schemas must match between multiple inputs t2 = (col1=[1.2, 2.3, 3.4, 4.5, 5.6],) @test_throws ArgumentError Arrow.Table([Arrow.tobuffer(t), Arrow.tobuffer(t2)]) # test multiple inputs treated as one table tbls = collect(Arrow.Stream([Arrow.tobuffer(t), Arrow.tobuffer(t)])) @test tbls[1].col1 == tbls[2].col1 @test tbls[1].col2 == tbls[2].col2 # schemas must match between multiple inputs t2 = (col1=[1.2, 2.3, 3.4, 4.5, 5.6],) @test_throws ArgumentError collect( Arrow.Stream([Arrow.tobuffer(t), Arrow.tobuffer(t2)]), ) end @testset "# 253" begin # https://github.com/apache/arrow-julia/issues/253 @test Arrow.toidict(Pair{String,String}[]) == Base.ImmutableDict{String,String}() end @testset "# 232" begin # https://github.com/apache/arrow-julia/issues/232 t = (; x=[Dict(true => 1.32, 1.2 => 0.53495216)]) @test_throws ArgumentError( "`keytype(d)` must be concrete to serialize map-like `d`, but `keytype(d) == Real`", ) Arrow.tobuffer(t) t = (; x=[Dict(32.0 => true, 1.2 => 0.53495216)]) @test_throws ArgumentError( "`valtype(d)` must be concrete to serialize map-like `d`, but `valtype(d) == Real`", ) Arrow.tobuffer(t) t = (; x=[Dict(true => 1.32, 1.2 => true)]) @test_throws ArgumentError( "`keytype(d)` must be concrete to serialize map-like `d`, but `keytype(d) == Real`", ) Arrow.tobuffer(t) end @testset "# 214" begin # https://github.com/apache/arrow-julia/issues/214 t1 = (; x=[(Nanosecond(42),)]) t2 = Arrow.Table(Arrow.tobuffer(t1)) t3 = Arrow.Table(Arrow.tobuffer(t2)) @test t3.x == t1.x t1 = (; x=[(; a=Nanosecond(i), b=Nanosecond(i + 1)) for i = 1:5]) t2 = Arrow.Table(Arrow.tobuffer(t1)) t3 = Arrow.Table(Arrow.tobuffer(t2)) @test t3.x == t1.x end @testset "Writer" begin io = IOBuffer() writer = open(Arrow.Writer, io) a = 1:26 b = 'A':'Z' partitionsize = 10 iter_a = Iterators.partition(a, partitionsize) iter_b = Iterators.partition(b, partitionsize) for (part_a, part_b) in zip(iter_a, iter_b) Arrow.write(writer, (a=part_a, b=part_b)) end close(writer) seekstart(io) table = Arrow.Table(io) @test table.a == collect(a) @test table.b == collect(b) end @testset "# Empty input" begin @test Arrow.Table(UInt8[]) isa Arrow.Table @test isempty(Tables.rows(Arrow.Table(UInt8[]))) @test Arrow.Stream(UInt8[]) isa Arrow.Stream @test isempty(Tables.partitions(Arrow.Stream(UInt8[]))) end @testset "# 324" begin # https://github.com/apache/arrow-julia/issues/324 @test_throws ArgumentError filter!(x -> x > 1, Arrow.toarrowvector([1, 2, 3])) end @testset "# 327" begin # https://github.com/apache/arrow-julia/issues/327 zdt = ZonedDateTime(DateTime(2020, 11, 1, 6), tz"America/New_York"; from_utc=true) arrow_zdt = ArrowTypes.toarrow(zdt) zdt_again = ArrowTypes.fromarrow(ZonedDateTime, arrow_zdt) @test zdt == zdt_again # Check that we still correctly read in old TimeZones original_table = (; col=[ZonedDateTime(DateTime(1, 2, 3, 4, 5, 6), tz"UTC+3") for _ = 1:5]) table = Arrow.Table(joinpath(@__DIR__, "old_zdt.arrow")) @test original_table.col == table.col end @testset "# 243" begin table = (; col=[(; v=v"1"), (; v=v"2"), missing]) @test isequal(Arrow.Table(Arrow.tobuffer(table)).col, table.col) end @testset "# 367" begin t = (; x=Union{ZonedDateTime,Missing}[missing]) a = Arrow.Table(Arrow.tobuffer(t)) @test Tables.schema(a) == Tables.schema(t) @test isequal(a.x, t.x) end # https://github.com/apache/arrow-julia/issues/414 df = DataFrame(("$i" => rand(1000) for i = 1:65536)...) df_load = Arrow.Table(Arrow.tobuffer(df)) @test Tables.schema(df) == Tables.schema(df_load) for (col1, col2) in zip(Tables.columns(df), Tables.columns(df_load)) @test col1 == col2 end @testset "# 411" begin # Vector{UInt8} are written as List{UInt8} in Arrow # Base.CodeUnits are written as Binary t = ( a=[[0x00, 0x01], UInt8[], [0x03]], am=[[0x00, 0x01], [0x03], missing], b=[b"01", b"", b"3"], bm=[b"01", b"3", missing], c=["a", "b", "c"], cm=["a", "c", missing], ) buf = Arrow.tobuffer(t) tt = Arrow.Table(buf) @test t.a == tt.a @test isequal(t.am, tt.am) @test t.b == tt.b @test isequal(t.bm, tt.bm) @test t.c == tt.c @test isequal(t.cm, tt.cm) @test Arrow.schema(tt)[].fields[1].type isa Arrow.Flatbuf.List @test Arrow.schema(tt)[].fields[3].type isa Arrow.Flatbuf.Binary pos = position(buf) Arrow.append(buf, tt) seekstart(buf) buf1 = read(buf, pos) buf2 = read(buf) t1 = Arrow.Table(buf1) t2 = Arrow.Table(buf2) @test isequal(t1.a, t2.a) @test isequal(t1.am, t2.am) @test isequal(t1.b, t2.b) @test isequal(t1.bm, t2.bm) @test isequal(t1.c, t2.c) @test isequal(t1.cm, t2.cm) end @testset "# 435" begin t = Arrow.Table( joinpath(dirname(pathof(Arrow)), "../test/java_compress_len_neg_one.arrow"), ) @test length(t) == 15 @test length(t.isA) == 102 end @testset "# 293" begin t = (a=[1, 2, 3], b=[1.0, 2.0, 3.0]) buf = Arrow.tobuffer(t) tbl = Arrow.Table(buf) parts = Tables.partitioner((t, t)) buf2 = Arrow.tobuffer(parts) tbl2 = Arrow.Table(buf2) for t in Tables.partitions(tbl2) @test t.a == tbl.a @test t.b == tbl.b end end @testset "# 437" begin t = Arrow.Table( joinpath( dirname(pathof(Arrow)), "../test/java_compressed_zero_length.arrow", ), ) @test length(t) == 2 @test length(t.name) == 0 end @testset "# 458" begin x = (; a=[[[[1]]]]) buf = Arrow.tobuffer(x) t = Arrow.Table(buf) @test t.a[1][1][1][1] == 1 end @testset "# 456" begin NT = @NamedTuple{x::Int, y::Union{Missing,Int}} data = NT[(x=1, y=2), (x=2, y=missing), (x=3, y=4), (x=4, y=5)] t = [(a=1, b=view(data, 1:2)), (a=2, b=view(data, 3:4)), missing] @test Arrow.toarrowvector(t) isa Arrow.Struct end # @testset "# 461" begin # table = (; v=[v"1", v"2", missing]) # buf = Arrow.tobuffer(table) # table2 = Arrow.Table(buf) # @test isequal(table.v, table2.v) # end if isdefined(ArrowTypes, :StructElement) @testset "# 493" begin # This test stresses the existence of the mechanism # implemented in https://github.com/apache/arrow-julia/pull/493, # but doesn't stress the actual use case that motivates # that mechanism, simply because it'd be more annoying to # write that test; see the PR for details. struct Foo493 x::Int y::Int end ArrowTypes.arrowname(::Type{Foo493}) = Symbol("JuliaLang.Foo493") ArrowTypes.JuliaType(::Val{Symbol("JuliaLang.Foo493")}, T) = Foo493 function ArrowTypes.fromarrowstruct( ::Type{Foo493}, ::Val{fnames}, x..., ) where {fnames} nt = NamedTuple{fnames}(x) return Foo493(nt.x + 1, nt.y + 1) end t = (; f=[Foo493(1, 2), Foo493(3, 4)]) buf = Arrow.tobuffer(t) tbl = Arrow.Table(buf) @test tbl.f[1] === Foo493(2, 3) @test tbl.f[2] === Foo493(4, 5) end end @testset "# 504" begin struct Foo504 x::Int end struct Bar504 a::Foo504 end v = [Bar504(Foo504(i)) for i = 1:3] io = IOBuffer() Arrow.write(io, v; file=false) seekstart(io) Arrow.append(io, v) # testing the compatility between the schema of the arrow Table, and the "schema" of v (using the fallback mechanism of Tables.jl) seekstart(io) t = Arrow.Table(io) @test Arrow.Tables.rowcount(t) == 6 end @testset "# 526: Arrow.Time" begin tt = testtables[4] # just to make sure we're grabbing the correct table @test first(tt) == "arrow date/time types" tbl = Arrow.Table(Arrow.tobuffer(tt[2])) @test tbl.col16[1] == Dates.Time(0, 0, 0) end @testset "#511: Bug in reading Utf8View data" begin t = Arrow.Table( joinpath(dirname(pathof(Arrow)), "../test/reject_reason_trimmed.arrow"), ) @test t.reject_reason[end] == "POST_ONLY" end end # @testset "misc" @testset "DataAPI.metadata" begin df = DataFrame(a=1, b=2, c=3) for i = 1:2 io = IOBuffer() if i == 1 # skip writing metadata in the first iteration Arrow.write(io, df) else Arrow.write(io, df, metadata=metadata(df), colmetadata=colmetadata(df)) end seekstart(io) tbl = Arrow.Table(io) @test DataAPI.metadatasupport(typeof(tbl)) == (read=true, write=false) @test metadata(tbl) == metadata(df) @test metadata(tbl; style=true) == metadata(df; style=true) @test_throws Exception metadata(tbl, "xyz") @test metadata(tbl, "xyz", "something") == "something" @test metadata(tbl, "xyz", "something"; style=true) == ("something", :default) @test metadatakeys(tbl) == metadatakeys(df) @test DataAPI.colmetadatasupport(typeof(tbl)) == (read=true, write=false) @test colmetadata(tbl) == colmetadata(df) @test colmetadata(tbl; style=true) == colmetadata(df; style=true) @test_throws MethodError colmetadata(tbl, "xyz") @test_throws KeyError colmetadata(tbl, :xyz) @test colmetadata(tbl, :b) == colmetadata(df, :b) @test_throws MethodError colmetadata(tbl, :b, "xyz") @test colmetadata(tbl, :b, "xyz", "something") == "something" @test colmetadata(tbl, :b, "xyz", "something"; style=true) == ("something", :default) @test Set(colmetadatakeys(tbl)) == Set(colmetadatakeys(df)) # add metadata for the second iteration metadata!(df, "tkey", "tvalue") metadata!(df, "tkey2", "tvalue2") colmetadata!(df, :a, "ackey", "acvalue") colmetadata!(df, :a, "ackey2", "acvalue2") colmetadata!(df, :c, "cckey", "ccvalue") end end # @testset "DataAPI.metadata" end ================================================ FILE: test/testappend.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. function testappend(nm, t, writekw, readkw, extratests) @testset "append: $nm" begin io = Arrow.tobuffer(t; writekw...) bytes = read(io) mktemp() do path, io write(io, bytes) close(io) t1 = Arrow.Table(read(path); readkw...) f1 = first(Tables.columns(t1)) Arrow.append(path, t1; writekw..., readkw...) nparts = 0 for t2 in Arrow.Stream(path) @test isequal(f1, first(Tables.columns(t2))) nparts += 1 end @test nparts == 2 end end end function testappend_compression(compression_option) mktempdir() do path testdata = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) file1 = joinpath(path, "table1.arrow") file2 = joinpath(path, "table2.arrow") open(file1, "w") do io Arrow.write(io, testdata; file=false, compress=compression_option) end isstream, schema, compression = open(Arrow.stream_properties, file1) @test isstream @test compression == compression_option open(file2, "w") do io Arrow.write(io, testdata; file=false) end arrow_table2 = Arrow.Table(file2) arrow_table2 |> Arrow.append(file1) arrow_table1 = Arrow.Table(file1) isstream, schema, compression = open(Arrow.stream_properties, file1) @test isstream @test compression == compression_option @test length(Tables.columns(arrow_table1)[1]) == 20 @test length(Tables.columns(arrow_table2)[1]) == 10 end end function testappend_partitions() mktempdir() do path testdata = (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) file1 = joinpath(path, "table1.arrow") file2 = joinpath(path, "table2.arrow") open(file1, "w") do io Arrow.write(io, testdata; file=false) end arrow_table1 = Arrow.Table(file1) isstream, schema, compression = open(Arrow.stream_properties, file1) @test isstream @test compression === nothing @test schema.names == (:col1,) @test schema.types == (Int64,) # can only append to arrow stream open(file2, "w") do io Arrow.write(io, testdata; file=true) end @test_throws ArgumentError Arrow.append(file2, arrow_table1) # can append to an empty file rm(file2) for _ = 1:5 Arrow.append(file2, arrow_table1) end appended_table1 = Arrow.Table(file2) @test length(Tables.columns(appended_table1)[1]) == 50 # schema must match testdata2 = (col2=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],) open(file2, "w") do io Arrow.write(io, testdata2; file=false) end @test_throws ArgumentError Arrow.append(file2, arrow_table1) # recreate file2 in arrow format with correct schema open(file2, "w") do io Arrow.write(io, testdata; file=false) end # start # arrow_table1: 1 partition, 10 rows # arrow_table2: 1 partition, 10 rows arrow_table2 = Arrow.Table(file2) @test length(Tables.columns(arrow_table1)[1]) == 10 @test length(Tables.columns(arrow_table2)[1]) == 10 @test_throws ArgumentError Arrow.append(file1, arrow_table2; ntasks=-1) arrow_table2 |> Arrow.append(file1) arrow_table1 = Arrow.Table(file1) # now # arrow_table1: 2 partitions, 20 rows # arrow_table2: 1 partition, 10 rows @test Tables.schema(arrow_table1) == Tables.schema(arrow_table2) @test length(Tables.columns(arrow_table1)[1]) == 20 @test length(Tables.columns(arrow_table2)[1]) == 10 @test length(collect(Tables.partitions(Arrow.Stream(file1)))) == 2 * length(collect(Tables.partitions(Arrow.Stream(file2)))) Arrow.append(file2, arrow_table1; ntasks=1) # append with single task arrow_table2 = Arrow.Table(file2) # now # arrow_table1: 2 partitions, 20 rows # arrow_table2: 2 partitions, 30 rows (both partitions of table1 are appended as separate partitions) @test Tables.schema(arrow_table1) == Tables.schema(arrow_table2) @test length(Tables.columns(arrow_table1)[1]) == 20 @test length(Tables.columns(arrow_table2)[1]) == 30 @test length(collect(Tables.partitions(Arrow.Stream(file1)))) == 2 @test length(collect(Tables.partitions(Arrow.Stream(file2)))) == 3 Arrow.append(file1, Arrow.Stream(file2)) arrow_table1 = Arrow.Table(file1) # now # arrow_table1: 4 partitions, 50 rows (partitions of file2 stream are appended without being merged) # arrow_table2: 2 partitions, 30 rows @test Tables.schema(arrow_table1) == Tables.schema(arrow_table2) @test length(Tables.columns(arrow_table1)[1]) == 50 @test length(Tables.columns(arrow_table2)[1]) == 30 @test length(collect(Tables.partitions(Arrow.Stream(file1)))) == 5 @test length(collect(Tables.partitions(Arrow.Stream(file2)))) == 3 end end ================================================ FILE: test/testtables.jl ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. testtables = [ ( "basic", (col1=Int64[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],), NamedTuple(), NamedTuple(), nothing, ), ( "missing values", (col1=Union{Int64,Missing}[1, 2, 3, 4, 5, 6, 7, 8, 9, missing],), NamedTuple(), NamedTuple(), nothing, ), ( "primitive types", ( col1=[missing, missing, missing, missing], col2=Union{UInt8,Missing}[0, 1, 2, missing], col3=Union{UInt16,Missing}[0, 1, 2, missing], col4=Union{UInt32,Missing}[0, 1, 2, missing], col5=Union{UInt64,Missing}[0, 1, 2, missing], col6=Union{Int8,Missing}[0, 1, 2, missing], col7=Union{Int16,Missing}[0, 1, 2, missing], col8=Union{Int32,Missing}[0, 1, 2, missing], col9=Union{Int64,Missing}[0, 1, 2, missing], col10=Union{Float16,Missing}[0, 1, 2, missing], col11=Union{Float32,Missing}[0, 1, 2, missing], col12=Union{Float64,Missing}[0, 1, 2, missing], col13=[true, false, true, missing], ), NamedTuple(), NamedTuple(), nothing, ), ( "arrow date/time types", ( col14=[ zero(Arrow.Decimal{Int32(2),Int32(2),Int128}), zero(Arrow.Decimal{Int32(2),Int32(2),Int128}), zero(Arrow.Decimal{Int32(2),Int32(2),Int128}), missing, ], col15=[ zero(Arrow.Date{Arrow.Meta.DateUnit.DAY,Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY,Int32}), zero(Arrow.Date{Arrow.Meta.DateUnit.DAY,Int32}), missing, ], col16=[ zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND,Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND,Int32}), zero(Arrow.Time{Arrow.Meta.TimeUnit.SECOND,Int32}), missing, ], col17=[ zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND,nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND,nothing}), zero(Arrow.Timestamp{Arrow.Meta.TimeUnit.SECOND,nothing}), missing, ], col18=[ zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH,Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH,Int32}), zero(Arrow.Interval{Arrow.Meta.IntervalUnit.YEAR_MONTH,Int32}), missing, ], col19=[ zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), zero(Arrow.Duration{Arrow.Meta.TimeUnit.SECOND}), missing, ], col20=[ zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND,Int64}), zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND,Int64}), zero(Arrow.Date{Arrow.Meta.DateUnit.MILLISECOND,Int64}), missing, ], ), NamedTuple(), (convert=false,), nothing, ), ( "list types", ( col1=Union{String,Missing}["hey", "there", "sailor", missing], col2=Union{Vector{UInt8},Missing}[b"hey", b"there", b"sailor", missing], col3=Union{Vector{Int64},Missing}[Int64[1], Int64[2], Int64[3], missing], col4=Union{NTuple{2,Vector{Int64}},Missing}[ (Int64[1], Int64[2]), missing, missing, (Int64[3], Int64[4]), ], col5=Union{NTuple{2,UInt8},Missing}[ (0x01, 0x02), (0x03, 0x04), missing, (0x05, 0x06), ], col6=NamedTuple{(:a, :b),Tuple{Int64,String}}[ (a=Int64(1), b="hey"), (a=Int64(2), b="there"), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob"), ], ), NamedTuple(), NamedTuple(), nothing, ), ("empty list types", (col1=[[]], col2=[()]), NamedTuple(), NamedTuple(), nothing), ( "unions", ( col1=Arrow.DenseUnionVector( Union{Int64,Float64,Missing}[1, 2.0, 3, 4.0, missing], ), col2=Arrow.SparseUnionVector( Union{Int64,Float64,Missing}[1, 2.0, 3, 4.0, missing], ), ), NamedTuple(), NamedTuple(), nothing, ), ( "dict encodings", (col1=Arrow.DictEncode(Int64[4, 5, 6]),), NamedTuple(), NamedTuple(), function (tt) col1 = copy(tt.col1) @test typeof(col1) == PooledVector{Int64,Int8,Vector{Int8}} end, ), ( "more dict encodings", ( col1=Arrow.DictEncode( NamedTuple{(:a, :b),Tuple{Int64,Union{String,Missing}}}[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob"), ], ), ), NamedTuple(), NamedTuple(), nothing, ), ("PooledArray", (col1=PooledArray([4, 5, 6, 6]),), NamedTuple(), NamedTuple(), nothing), ( "auto-converting types", ( col1=[Date(2001, 1, 2), Date(2010, 10, 10), Date(2020, 12, 1)], col2=[Time(1, 1, 2), Time(13, 10, 10), Time(22, 12, 1)], col3=[DateTime(2001, 1, 2), DateTime(2010, 10, 10), DateTime(2020, 12, 1)], col4=[ ZonedDateTime(2001, 1, 2, TimeZone("America/Denver")), ZonedDateTime(2010, 10, 10, TimeZone("America/Denver")), ZonedDateTime(2020, 12, 1, TimeZone("America/Denver")), ], ), NamedTuple(), NamedTuple(), nothing, ), ( "Map", (col1=[Dict(Int32(1) => Float32(3.14)), missing],), NamedTuple(), NamedTuple(), nothing, ), ( "non-standard types", ( col1=[:hey, :there, :sailor], col2=['a', 'b', 'c'], col3=Arrow.DictEncode(['a', 'a', 'b']), col4=[ UUID("48075322-8645-4ac6-b590-c9f46068565a"), UUID("99c7d976-ccfd-45b9-9793-51008607c638"), UUID("f96d9974-5a7b-47e3-bbc0-d680d11490d4"), ], ), NamedTuple(), NamedTuple(), nothing, ), ( "large lists", ( col1=Union{String,Missing}["hey", "there", "sailor", missing], col2=Union{Vector{UInt8},Missing}[b"hey", b"there", b"sailor", missing], col3=Union{Vector{Int64},Missing}[Int64[1], Int64[2], Int64[3], missing], col4=Union{NTuple{2,Vector{Int64}},Missing}[ (Int64[1], Int64[2]), missing, missing, (Int64[3], Int64[4]), ], col5=Union{NTuple{2,UInt8},Missing}[ (0x01, 0x02), (0x03, 0x04), missing, (0x05, 0x06), ], col6=NamedTuple{(:a, :b),Tuple{Int64,String}}[ (a=Int64(1), b="hey"), (a=Int64(2), b="there"), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob"), ], ), (largelists=true,), NamedTuple(), nothing, ), ( "dictencode keyword", ( col1=Int64[1, 2, 3, 4], col2=Union{String,Missing}["hey", "there", "sailor", missing], col3=Arrow.DictEncode( NamedTuple{(:a, :b),Tuple{Int64,Union{String,Missing}}}[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b="sailor"), (a=Int64(4), b="jo-bob"), ], ), col4=[:a, :b, :c, missing], col5=[Date(2020, 1, 1) for x = 1:4], ), (dictencode=true,), NamedTuple(), nothing, ), ( "nesteddictencode keyword", ( col1=NamedTuple{ (:a, :b), Tuple{Int64,Union{Missing,NamedTuple{(:c,),Tuple{String}}}}, }[ (a=Int64(1), b=missing), (a=Int64(1), b=missing), (a=Int64(3), b=(c="sailor",)), (a=Int64(4), b=(c="jo-bob",)), ], ), (dictencode=true, dictencodenested=true), NamedTuple(), nothing, ), ( "Julia unions", ( col1=Union{Int,String}[1, "hey", 2, "ho"], col2=Union{Char,NamedTuple{(:a,),Tuple{Symbol}}}['a', (a=:hey,), 'b', (a=:ho,)], ), (denseunions=false,), NamedTuple(), nothing, ), ( "Decimal256", ( col1=[ zero(Arrow.Decimal{Int32(2),Int32(2),Arrow.Int256}), zero(Arrow.Decimal{Int32(2),Int32(2),Arrow.Int256}), zero(Arrow.Decimal{Int32(2),Int32(2),Arrow.Int256}), missing, ], ), NamedTuple(), (convert=false,), nothing, ), ]; function testtable(nm, t, writekw, readkw, extratests) @testset "testing: $nm" begin io = Arrow.tobuffer(t; writekw...) tt = Arrow.Table(io; readkw...) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) extratests !== nothing && extratests(tt) seekstart(io) str = Arrow.Stream(io; readkw...) tt = first(str) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) # compressed io = Arrow.tobuffer(t; compress=((:lz4, :zstd)[rand(1:2)]), writekw...) tt = Arrow.Table(io; readkw...) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) extratests !== nothing && extratests(tt) seekstart(io) str = Arrow.Stream(io; readkw...) tt = first(str) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) # file io = Arrow.tobuffer(t; file=true, writekw...) tt = Arrow.Table(io; readkw...) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) extratests !== nothing && extratests(tt) seekstart(io) str = Arrow.Stream(io; readkw...) tt = first(str) @test length(tt) == length(t) @test all(isequal.(values(t), values(tt))) end end