[ { "path": ".github/release-drafter.yml", "content": "name-template: 'v$NEXT_MINOR_VERSION'\ntag-template: 'v$NEXT_MINOR_VERSION'\ncategories:\n - title: 'Added'\n labels:\n - 'feature'\n - title: 'Changed'\n labels:\n - 'enhancement'\n - 'dependency-update'\n - title: 'Fixed'\n labels:\n - 'fix'\n - 'bug'\ninclude-labels:\n - 'feature'\n - 'enhancement'\n - 'dependency-update'\n - 'fix'\n - 'bug'\nexclude-labels:\n - 'skip-changelog'\n - 'documentation'\n - 'build/process improvement'\nchange-template: '- $TITLE [#$NUMBER](https://github.com/typelevel/frameless/pull/$NUMBER) (@$AUTHOR)'\ntemplate: |\n $CHANGES\n" }, { "path": ".github/workflows/ci.yml", "content": "# This file was automatically generated by sbt-github-actions using the\n# githubWorkflowGenerate task. You should add and commit this file to\n# your git repository. It goes without saying that you shouldn't edit\n# this file by hand! Instead, if you wish to make changes, you should\n# change your sbt build configuration to revise the workflow description\n# to meet your needs, then regenerate this file.\n\nname: Continuous Integration\n\non:\n pull_request:\n branches: ['**', '!update/**', '!pr/**']\n push:\n branches: ['**', '!update/**', '!pr/**']\n tags: [v*]\n\nenv:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n SBT_OPTS: '-Xms1g -Xmx4g'\n SPARK_LOCAL_IP: localhost\n\n\nconcurrency:\n group: ${{ github.workflow }} @ ${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n build:\n name: Test\n strategy:\n matrix:\n os: [ubuntu-22.04]\n scala: [2.13, 2.12]\n java: [temurin@8]\n project: [root-spark33, root-spark34, root-spark35]\n exclude:\n - scala: 2.13\n project: root-spark33\n - scala: 2.13\n project: root-spark34\n runs-on: ${{ matrix.os }}\n timeout-minutes: 60\n steps:\n - name: Checkout current branch (full)\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Setup sbt\n uses: sbt/setup-sbt@v1\n\n - name: Setup Java (temurin@8)\n id: setup-java-temurin-8\n if: matrix.java == 'temurin@8'\n uses: actions/setup-java@v5\n with:\n distribution: temurin\n java-version: 8\n cache: sbt\n\n - name: sbt update\n if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'\n run: sbt +update\n\n - name: Check that workflows are up to date\n run: sbt githubWorkflowCheck\n\n - name: Check formatting\n if: matrix.java == 'temurin@8' && matrix.os == 'ubuntu-22.04'\n run: sbt '++ ${{ matrix.scala }}' 'project ${{ matrix.project }}' scalafmtCheckAll 'project /' scalafmtSbtCheck\n\n - name: Test & Compute Coverage\n run: sbt '++ ${{ matrix.scala }}' 'project ${{ matrix.project }}' coverage test test/coverageReport\n\n - name: Check binary compatibility\n if: matrix.java == 'temurin@8' && matrix.os == 'ubuntu-22.04'\n run: sbt '++ ${{ matrix.scala }}' 'project ${{ matrix.project }}' mimaReportBinaryIssues\n\n - name: Generate API documentation\n if: matrix.java == 'temurin@8' && matrix.os == 'ubuntu-22.04'\n run: sbt '++ ${{ matrix.scala }}' 'project ${{ matrix.project }}' doc\n\n - uses: codecov/codecov-action@v3\n with:\n flags: ${{ matrix.scala }}-${{ matrix.project }}\n\n publish:\n name: Publish Artifacts\n needs: [build]\n if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master')\n strategy:\n matrix:\n os: [ubuntu-22.04]\n java: [temurin@8]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Checkout current branch (full)\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Setup sbt\n uses: sbt/setup-sbt@v1\n\n - name: Setup Java (temurin@8)\n id: setup-java-temurin-8\n if: matrix.java == 'temurin@8'\n uses: actions/setup-java@v5\n with:\n distribution: temurin\n java-version: 8\n cache: sbt\n\n - name: sbt update\n if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'\n run: sbt +update\n\n - name: Import signing key\n if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == ''\n env:\n PGP_SECRET: ${{ secrets.PGP_SECRET }}\n PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}\n run: echo $PGP_SECRET | base64 -d -i - | gpg --import\n\n - name: Import signing key and strip passphrase\n if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != ''\n env:\n PGP_SECRET: ${{ secrets.PGP_SECRET }}\n PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}\n run: |\n echo \"$PGP_SECRET\" | base64 -d -i - > /tmp/signing-key.gpg\n echo \"$PGP_PASSPHRASE\" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg\n (echo \"$PGP_PASSPHRASE\"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1)\n\n - name: Publish\n env:\n SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}\n SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}\n SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }}\n run: sbt tlCiRelease\n\n dependency-submission:\n name: Submit Dependencies\n if: github.event.repository.fork == false && github.event_name != 'pull_request'\n strategy:\n matrix:\n os: [ubuntu-22.04]\n java: [temurin@8]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Checkout current branch (full)\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Setup sbt\n uses: sbt/setup-sbt@v1\n\n - name: Setup Java (temurin@8)\n id: setup-java-temurin-8\n if: matrix.java == 'temurin@8'\n uses: actions/setup-java@v5\n with:\n distribution: temurin\n java-version: 8\n cache: sbt\n\n - name: sbt update\n if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'\n run: sbt +update\n\n - name: Submit Dependencies\n uses: scalacenter/sbt-dependency-submission@v2\n with:\n modules-ignore: root-spark33_2.13 root-spark33_2.12 docs_2.13 docs_2.12 root-spark34_2.13 root-spark34_2.12 root-spark35_2.13 root-spark35_2.12\n configs-ignore: test scala-tool scala-doc-tool test-internal\n\n site:\n name: Generate Site\n strategy:\n matrix:\n os: [ubuntu-22.04]\n java: [temurin@11]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Checkout current branch (full)\n uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Setup sbt\n uses: sbt/setup-sbt@v1\n\n - name: Setup Java (temurin@8)\n id: setup-java-temurin-8\n if: matrix.java == 'temurin@8'\n uses: actions/setup-java@v5\n with:\n distribution: temurin\n java-version: 8\n cache: sbt\n\n - name: sbt update\n if: matrix.java == 'temurin@8' && steps.setup-java-temurin-8.outputs.cache-hit == 'false'\n run: sbt +update\n\n - name: Setup Java (temurin@11)\n id: setup-java-temurin-11\n if: matrix.java == 'temurin@11'\n uses: actions/setup-java@v5\n with:\n distribution: temurin\n java-version: 11\n cache: sbt\n\n - name: sbt update\n if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false'\n run: sbt +update\n\n - name: Generate site\n run: sbt docs/tlSite\n\n - name: Publish site\n if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master'\n uses: peaceiris/actions-gh-pages@v4.0.0\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n publish_dir: mdocs/target/docs/site\n keep_files: true\n" }, { "path": ".github/workflows/clean.yml", "content": "# This file was automatically generated by sbt-github-actions using the\n# githubWorkflowGenerate task. You should add and commit this file to\n# your git repository. It goes without saying that you shouldn't edit\n# this file by hand! Instead, if you wish to make changes, you should\n# change your sbt build configuration to revise the workflow description\n# to meet your needs, then regenerate this file.\n\nname: Clean\n\non: push\n\njobs:\n delete-artifacts:\n name: Delete Artifacts\n runs-on: ubuntu-latest\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n steps:\n - name: Delete artifacts\n run: |\n # Customize those three lines with your repository and credentials:\n REPO=${GITHUB_API_URL}/repos/${{ github.repository }}\n\n # A shortcut to call GitHub API.\n ghapi() { curl --silent --location --user _:$GITHUB_TOKEN \"$@\"; }\n\n # A temporary file which receives HTTP response headers.\n TMPFILE=/tmp/tmp.$$\n\n # An associative array, key: artifact name, value: number of artifacts of that name.\n declare -A ARTCOUNT\n\n # Process all artifacts on this repository, loop on returned \"pages\".\n URL=$REPO/actions/artifacts\n while [[ -n \"$URL\" ]]; do\n\n # Get current page, get response headers in a temporary file.\n JSON=$(ghapi --dump-header $TMPFILE \"$URL\")\n\n # Get URL of next page. Will be empty if we are at the last page.\n URL=$(grep '^Link:' \"$TMPFILE\" | tr ',' '\\n' | grep 'rel=\"next\"' | head -1 | sed -e 's/.*.*//')\n rm -f $TMPFILE\n\n # Number of artifacts on this page:\n COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') ))\n\n # Loop on all artifacts on this page.\n for ((i=0; $i < $COUNT; i++)); do\n\n # Get name of artifact and count instances of this name.\n name=$(jq <<<$JSON -r \".artifacts[$i].name?\")\n ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1))\n\n id=$(jq <<<$JSON -r \".artifacts[$i].id?\")\n size=$(( $(jq <<<$JSON -r \".artifacts[$i].size_in_bytes?\") ))\n printf \"Deleting '%s' #%d, %'d bytes\\n\" $name ${ARTCOUNT[$name]} $size\n ghapi -X DELETE $REPO/actions/artifacts/$id\n done\n done\n" }, { "path": ".github/workflows/release-drafter.yml", "content": "name: Release Drafter\n\non:\n push:\n branches:\n - master\n pull_request:\n types: [opened, reopened, synchronize]\n\njobs:\n update_release_draft:\n runs-on: ubuntu-latest\n steps:\n - uses: release-drafter/release-drafter@v5.15.0\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "*.class\n*.log\n\n# sbt specific\n.bsp/\ndist/*\ntarget/\nlib_managed/\nsrc_managed/\nproject/boot/\nproject/plugins/project/\n\n# Scala-IDE specific\n.scala_dependencies\n.cache\n.classpath\n.project\n.worksheet/\nbin/\n.settings/\n.ensime\n.ensime_cache/\n\n# IntelliJ specific\n.idea\n\n# OS X\n.DS_Store\nnode_modules\n\n# VSCode\n.history\n.metals\n.vscode\n.bloop\nmetals.sbt\n" }, { "path": ".scalafmt.conf", "content": "version = 3.8.6\nrunner.dialect = scala213\n\nnewlines.beforeMultilineDef = keep\nnewlines.topLevelStatements = [before]\nnewlines.beforeCurlyLambdaParams = multilineWithCaseOnly\nnewlines.afterCurlyLambdaParams = squash\nnewlines.implicitParamListModifierForce = [after]\nnewlines.avoidForSimpleOverflow = [tooLong]\nnewlines.avoidInResultType = true\nnewlines.sometimesBeforeColonInMethodReturnType = false\nnewlines.beforeTypeBounds = keep\n\nverticalMultiline.atDefnSite = true\nverticalMultiline.arityThreshold = 10\n\nspaces.inImportCurlyBraces = true\n\nincludeCurlyBraceInSelectChains = false\nincludeNoParensInSelectChains = false\noptIn.breakChainOnFirstMethodDot = false\n\ndocstrings.style = Asterisk\ndocstrings.wrap = no\n\nliterals.long=Upper\nliterals.float=Upper\nliterals.double=Upper\n" }, { "path": "LICENSE", "content": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# Frameless\n\n[![Workflow Badge](https://github.com/typelevel/frameless/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/typelevel/frameless/actions/workflows/ci.yml)\n[![Codecov Badge](https://codecov.io/gh/typelevel/frameless/branch/master/graph/badge.svg)](https://codecov.io/gh/typelevel/frameless)\n[![Discord Badge](https://img.shields.io/badge/chat-on%20discord-46BC99)](https://discord.gg/ZDZsxWcBJt)\n[![Maven Badge](https://img.shields.io/maven-central/v/org.typelevel/frameless-core_2.12?color=blue)](https://search.maven.org/search?q=g:org.typelevel%20and%20frameless)\n[![Snapshots Badge](https://img.shields.io/nexus/s/https/s01.oss.sonatype.org/org.typelevel/frameless-core_2.12)](https://s01.oss.sonatype.org/content/repositories/snapshots/org/typelevel/frameless-core_2.12/)\n\nFrameless is a Scala library for working with [Spark](http://spark.apache.org/) using more expressive types.\nIt consists of the following modules:\n\n* `frameless-dataset` for a more strongly typed `Dataset`/`DataFrame` API\n* `frameless-ml` for a more strongly typed Spark ML API based on `frameless-dataset`\n* `frameless-cats` for using Spark's `RDD` API with [cats](https://github.com/typelevel/cats)\n\nNote that while Frameless is still getting off the ground, it is very possible that breaking changes will be\nmade for at least the next few versions.\n\nThe Frameless project and contributors support the\n[Typelevel](http://typelevel.org/) [Code of Conduct](http://typelevel.org/code-of-conduct.html) and want all its\nassociated channels (e.g. GitHub, Discord) to be a safe and friendly environment for contributing and learning.\n\n## Versions and dependencies\n\nThe compatible versions of [Spark](http://spark.apache.org/) and\n[cats](https://github.com/typelevel/cats) are as follows:\n\n| Frameless | Spark | Cats | Cats-Effect | Scala |\n|-----------|-----------------------------|----------|-------------|-------------|\n| 0.16.0 | 3.5.0 / 3.4.0 / 3.3.0 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.15.0 | 3.4.0 / 3.3.0 / 3.2.2 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.14.1 | 3.4.0 / 3.3.0 / 3.2.2 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.14.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.13.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.12.0 | 3.2.1 / 3.1.3 / 3.0.3 | 2.x | 3.x | 2.12 / 2.13 |\n| 0.11.1 | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 |\n| 0.11.0* | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 |\n| 0.10.1 | 3.1.0 | 2.x | 2.x | 2.12 |\n| 0.9.0 | 3.0.0 | 1.x | 1.x | 2.12 |\n| 0.8.0 | 2.4.0 | 1.x | 1.x | 2.11 / 2.12 |\n| 0.7.0 | 2.3.1 | 1.x | 1.x | 2.11 |\n| 0.6.1 | 2.3.0 | 1.x | 0.8 | 2.11 |\n| 0.5.2 | 2.2.1 | 1.x | 0.8 | 2.11 |\n| 0.4.1 | 2.2.0 | 1.x | 0.8 | 2.11 |\n| 0.4.0 | 2.2.0 | 1.0.0-IF | 0.4 | 2.11 |\n\n_\\* 0.11.0 has broken Spark 3.1.2 and 3.0.1 artifacts published._\n\nStarting 0.11 we introduced Spark cross published artifacts:\n\n* By default, frameless artifacts depend on the most recent Spark version\n* Suffix `-spark{major}{minor}` is added to artifacts that are released for the previous Spark version(s)\n\nArtifact names examples:\n\n* `frameless-dataset` (the latest Spark dependency)\n* `frameless-dataset-spark33` (Spark 3.3.x dependency)\n* `frameless-dataset-spark32` (Spark 3.2.x dependency)\n\nVersions 0.5.x and 0.6.x have identical features. The first is compatible with Spark 2.2.1 and the second with 2.3.0.\n\nThe **only** dependency of the `frameless-dataset` module is on [shapeless](https://github.com/milessabin/shapeless) 2.3.2.\nTherefore, depending on `frameless-dataset`, has a minimal overhead on your Spark's application jar.\nOnly the `frameless-cats` module depends on cats and cats-effect, so if you prefer to work just with `Datasets` and not with `RDD`s,\nyou may choose not to depend on `frameless-cats`.\n\nFrameless intentionally **does not** have a compile dependency on Spark.\nThis essentially allows you to use any version of Frameless with any version of Spark.\nThe aforementioned table simply provides the versions of Spark we officially compile\nand test Frameless with, but other versions may probably work as well.\n\n### Breaking changes in 0.9\n\n* Spark 3 introduces a new ExpressionEncoder approach, the schema for single value DataFrame's is now [\"value\"](https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala#L270) not \"_1\".\n\n## Why?\n\nFrameless introduces a new Spark API, called `TypedDataset`.\nThe benefits of using `TypedDataset` compared to the standard Spark `Dataset` API are as follows:\n\n* Typesafe columns referencing (e.g., no more runtime errors when accessing non-existing columns)\n* Customizable, typesafe encoders (e.g., if a type does not have an encoder, it should not compile)\n* Enhanced type signature for built-in functions (e.g., if you apply an arithmetic operation on a non-numeric column, you\nget a compilation error)\n* Typesafe casting and projections\n\nClick [here](http://typelevel.org/frameless/TypedDatasetVsSparkDataset.html) for a\ndetailed comparison of `TypedDataset` with Spark's `Dataset` API.\n\n## Documentation\n\n* [TypedDataset: Feature Overview](http://typelevel.org/frameless/FeatureOverview.html)\n* [Typed Spark ML](http://typelevel.org/frameless/TypedML.html)\n* [Comparing TypedDatasets with Spark's Datasets](http://typelevel.org/frameless/TypedDatasetVsSparkDataset.html)\n* [Typed Encoders in Frameless](http://typelevel.org/frameless/TypedEncoder.html)\n* [Injection: Creating Custom Encoders](http://typelevel.org/frameless/Injection.html)\n* [Job\\[A\\]](http://typelevel.org/frameless/Job.html)\n* [Using Cats with RDDs](http://typelevel.org/frameless/Cats.html)\n* [Proof of Concept: TypedDataFrame](http://typelevel.org/frameless/TypedDataFrame.html)\n\n## Quick Start\n\nSince the 0.9.x release, Frameless is compiled only against Scala 2.12.x.\n\nTo use Frameless in your project add the following in your `build.sbt` file as needed:\n\n```scala\nval framelessVersion = \"\"\n\nresolvers ++= Seq(\n // for snapshot artifacts only\n \"s01-oss-sonatype\" at \"https://s01.oss.sonatype.org/content/repositories/snapshots\"\n)\n\nlibraryDependencies ++= List(\n \"org.typelevel\" %% \"frameless-dataset\" % framelessVersion,\n \"org.typelevel\" %% \"frameless-ml\" % framelessVersion,\n \"org.typelevel\" %% \"frameless-cats\" % framelessVersion\n)\n```\n\nAn easy way to bootstrap a Frameless sbt project:\n\n* if you have [Giter8][g8] installed then simply:\n\n```bash\ng8 imarios/frameless.g8\n```\n\n- with sbt >= 0.13.13:\n\n```bash\nsbt new imarios/frameless.g8\n```\n\nTyping `sbt console` inside your project will bring up a shell with Frameless\nand all its dependencies loaded (including Spark).\n\n## Need help?\n\nFeel free to messages us on our [discord](https://discord.gg/ZDZsxWcBJt)\nchannel for any issues/questions.\n\n## Development\n\nWe require at least _one_ sign-off (thumbs-up, +1, or similar) to merge pull requests. The current maintainers\n(people who can merge pull requests) are:\n\n* [adelbertc](https://github.com/adelbertc)\n* [imarios](https://github.com/imarios)\n* [kanterov](https://github.com/kanterov)\n* [non](https://github.com/non)\n* [OlivierBlanvillain](https://github.com/OlivierBlanvillain/)\n\n### Testing\n\nFrameless contains several property tests. To avoid `OutOfMemoryError`s, we\ntune the default generator sizes. The following environment variables may\nbe set to adjust the size of generated collections in the `TypedDataSet` suite:\n\n| Property | Default |\n|-----------------------------|--------:|\n| FRAMELESS_GEN_MIN_SIZE | 0 |\n| FRAMELESS_GEN_SIZE_RANGE | 20 |\n\n## License\n\nCode is provided under the Apache 2.0 license available at ,\nas well as in the LICENSE file. This is the same license used as Spark.\n\n[g8]: http://www.foundweekends.org/giter8/\n" }, { "path": "build.sbt", "content": "val sparkVersion = \"3.5.8\"\nval spark34Version = \"3.4.4\"\nval spark33Version = \"3.3.4\"\nval catsCoreVersion = \"2.13.0\"\nval catsEffectVersion = \"3.7.0\"\nval catsMtlVersion = \"1.6.0\"\nval scalatest = \"3.2.20\"\nval scalatestplus = \"3.1.0.0-RC2\"\nval shapeless = \"2.3.13\"\nval scalacheck = \"1.19.0\"\nval scalacheckEffect = \"2.1.0\"\nval refinedVersion = \"0.11.3\"\nval nakedFSVersion = \"0.1.0\"\n\nval Scala212 = \"2.12.20\"\nval Scala213 = \"2.13.18\"\n\nThisBuild / tlBaseVersion := \"0.16\"\n\nThisBuild / crossScalaVersions := Seq(Scala213, Scala212)\nThisBuild / scalaVersion := Scala212\nThisBuild / coverageScalacPluginVersion := \"2.3.0\"\n\nlazy val root = project\n .in(file(\".\"))\n .enablePlugins(NoPublishPlugin)\n .settings(crossScalaVersions := Nil)\n .aggregate(\n `root-spark35`,\n `root-spark34`,\n `root-spark33`,\n docs\n )\n\nlazy val `root-spark35` = project\n .in(file(\".spark35\"))\n .enablePlugins(NoPublishPlugin)\n .aggregate(core, cats, dataset, refined, ml)\n\nlazy val `root-spark34` = project\n .in(file(\".spark34\"))\n .enablePlugins(NoPublishPlugin)\n .aggregate(\n core,\n `cats-spark34`,\n `dataset-spark34`,\n `refined-spark34`,\n `ml-spark34`\n )\n\nlazy val `root-spark33` = project\n .in(file(\".spark33\"))\n .enablePlugins(NoPublishPlugin)\n .aggregate(\n core,\n `cats-spark33`,\n `dataset-spark33`,\n `refined-spark33`,\n `ml-spark33`\n )\n\nlazy val core =\n project.settings(name := \"frameless-core\").settings(framelessSettings)\n\nlazy val cats = project\n .settings(name := \"frameless-cats\")\n .settings(catsSettings)\n .dependsOn(dataset % \"test->test;compile->compile;provided->provided\")\n\nlazy val `cats-spark34` = project\n .settings(name := \"frameless-cats-spark34\")\n .settings(sourceDirectory := (cats / sourceDirectory).value)\n .settings(catsSettings)\n .settings(spark34Settings)\n .dependsOn(\n `dataset-spark34` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val `cats-spark33` = project\n .settings(name := \"frameless-cats-spark33\")\n .settings(sourceDirectory := (cats / sourceDirectory).value)\n .settings(catsSettings)\n .settings(spark33Settings)\n .dependsOn(\n `dataset-spark33` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val dataset = project\n .settings(name := \"frameless-dataset\")\n .settings(\n Compile / unmanagedSourceDirectories += baseDirectory.value / \"src\" / \"main\" / \"spark-3.4+\"\n )\n .settings(\n Test / unmanagedSourceDirectories += baseDirectory.value / \"src\" / \"test\" / \"spark-3.3+\"\n )\n .settings(datasetSettings)\n .settings(sparkDependencies(sparkVersion))\n .dependsOn(core % \"test->test;compile->compile\")\n\nlazy val `dataset-spark34` = project\n .settings(name := \"frameless-dataset-spark34\")\n .settings(sourceDirectory := (dataset / sourceDirectory).value)\n .settings(\n Compile / unmanagedSourceDirectories += (dataset / baseDirectory).value / \"src\" / \"main\" / \"spark-3.4+\"\n )\n .settings(\n Test / unmanagedSourceDirectories += (dataset / baseDirectory).value / \"src\" / \"test\" / \"spark-3.3+\"\n )\n .settings(datasetSettings)\n .settings(sparkDependencies(spark34Version))\n .settings(spark34Settings)\n .dependsOn(core % \"test->test;compile->compile\")\n\nlazy val `dataset-spark33` = project\n .settings(name := \"frameless-dataset-spark33\")\n .settings(sourceDirectory := (dataset / sourceDirectory).value)\n .settings(\n Compile / unmanagedSourceDirectories += (dataset / baseDirectory).value / \"src\" / \"main\" / \"spark-3\"\n )\n .settings(\n Test / unmanagedSourceDirectories += (dataset / baseDirectory).value / \"src\" / \"test\" / \"spark-3.3+\"\n )\n .settings(datasetSettings)\n .settings(sparkDependencies(spark33Version))\n .settings(spark33Settings)\n .dependsOn(core % \"test->test;compile->compile\")\n\nlazy val refined = project\n .settings(name := \"frameless-refined\")\n .settings(refinedSettings)\n .dependsOn(dataset % \"test->test;compile->compile;provided->provided\")\n\nlazy val `refined-spark34` = project\n .settings(name := \"frameless-refined-spark34\")\n .settings(sourceDirectory := (refined / sourceDirectory).value)\n .settings(refinedSettings)\n .settings(spark34Settings)\n .dependsOn(\n `dataset-spark34` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val `refined-spark33` = project\n .settings(name := \"frameless-refined-spark33\")\n .settings(sourceDirectory := (refined / sourceDirectory).value)\n .settings(refinedSettings)\n .settings(spark33Settings)\n .dependsOn(\n `dataset-spark33` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val ml = project\n .settings(name := \"frameless-ml\")\n .settings(mlSettings)\n .settings(sparkMlDependencies(sparkVersion))\n .dependsOn(\n core % \"test->test;compile->compile\",\n dataset % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val `ml-spark34` = project\n .settings(name := \"frameless-ml-spark34\")\n .settings(sourceDirectory := (ml / sourceDirectory).value)\n .settings(mlSettings)\n .settings(sparkMlDependencies(spark34Version))\n .settings(spark34Settings)\n .dependsOn(\n core % \"test->test;compile->compile\",\n `dataset-spark34` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val `ml-spark33` = project\n .settings(name := \"frameless-ml-spark33\")\n .settings(sourceDirectory := (ml / sourceDirectory).value)\n .settings(mlSettings)\n .settings(sparkMlDependencies(spark33Version))\n .settings(spark33Settings)\n .dependsOn(\n core % \"test->test;compile->compile\",\n `dataset-spark33` % \"test->test;compile->compile;provided->provided\"\n )\n\nlazy val docs = project\n .in(file(\"mdocs\"))\n .settings(framelessSettings)\n .settings(scalacOptions --= Seq(\"-Xfatal-warnings\", \"-Ywarn-unused-import\"))\n .enablePlugins(TypelevelSitePlugin)\n .settings(sparkDependencies(sparkVersion, Compile))\n .settings(sparkMlDependencies(sparkVersion, Compile))\n .settings(\n addCompilerPlugin(\n \"org.typelevel\" % \"kind-projector\" % \"0.13.4\" cross CrossVersion.full\n ),\n scalacOptions += \"-Ydelambdafy:inline\",\n libraryDependencies += \"org.typelevel\" %% \"mouse\" % \"1.3.2\"\n )\n .dependsOn(dataset, cats, ml)\n\ndef sparkDependencies(\n sparkVersion: String,\n scope: Configuration = Provided\n ) = Seq(\n libraryDependencies ++= Seq(\n \"org.apache.spark\" %% \"spark-core\" % sparkVersion % scope,\n \"org.apache.spark\" %% \"spark-sql\" % sparkVersion % scope\n )\n)\n\ndef sparkMlDependencies(sparkVersion: String, scope: Configuration = Provided) =\n Seq(\n libraryDependencies += \"org.apache.spark\" %% \"spark-mllib\" % sparkVersion % scope\n )\n\nlazy val catsSettings = framelessSettings ++ Seq(\n addCompilerPlugin(\n \"org.typelevel\" % \"kind-projector\" % \"0.13.4\" cross CrossVersion.full\n ),\n libraryDependencies ++= Seq(\n \"org.typelevel\" %% \"cats-core\" % catsCoreVersion,\n \"org.typelevel\" %% \"cats-effect\" % catsEffectVersion,\n \"org.typelevel\" %% \"cats-mtl\" % catsMtlVersion,\n \"org.typelevel\" %% \"alleycats-core\" % catsCoreVersion,\n \"org.typelevel\" %% \"scalacheck-effect\" % scalacheckEffect % Test\n )\n)\n\nlazy val datasetSettings =\n framelessSettings ++ framelessTypedDatasetREPL ++ Seq(\n mimaBinaryIssueFilters ++= {\n import com.typesafe.tools.mima.core._\n\n val imt = ProblemFilters.exclude[IncompatibleMethTypeProblem](_)\n val mc = ProblemFilters.exclude[MissingClassProblem](_)\n val dmm = ProblemFilters.exclude[DirectMissingMethodProblem](_)\n\n // TODO: Remove have version bump\n Seq(\n imt(\"frameless.TypedEncoder.mapEncoder\"),\n imt(\"frameless.TypedEncoder.arrayEncoder\"),\n imt(\"frameless.RecordEncoderFields.deriveRecordCons\"),\n imt(\"frameless.RecordEncoderFields.deriveRecordLast\"),\n mc(\"frameless.functions.FramelessLit\"),\n mc(f\"frameless.functions.FramelessLit$$\"),\n dmm(\"frameless.functions.package.litAggr\"),\n dmm(\"org.apache.spark.sql.FramelessInternals.column\")\n )\n },\n coverageExcludedPackages := \"org.apache.spark.sql.reflection\",\n libraryDependencies += \"com.globalmentor\" % \"hadoop-bare-naked-local-fs\" % nakedFSVersion % Test exclude (\n \"org.apache.hadoop\",\n \"hadoop-commons\"\n )\n )\n\nlazy val refinedSettings =\n framelessSettings ++ framelessTypedDatasetREPL ++ Seq(\n libraryDependencies += \"eu.timepit\" %% \"refined\" % refinedVersion\n )\n\nlazy val mlSettings = framelessSettings ++ framelessTypedDatasetREPL\n\nlazy val scalac212Options = Seq(\n \"-Xlint:-missing-interpolator,-unused,_\",\n \"-target:jvm-1.8\",\n \"-deprecation\",\n \"-encoding\",\n \"UTF-8\",\n \"-feature\",\n \"-unchecked\",\n \"-Xfatal-warnings\",\n \"-Yno-adapted-args\",\n \"-Ywarn-dead-code\",\n \"-Ywarn-numeric-widen\",\n \"-Ywarn-unused-import\",\n \"-Ywarn-value-discard\",\n \"-language:existentials\",\n \"-language:implicitConversions\",\n \"-language:higherKinds\",\n \"-Xfuture\",\n \"-Ypartial-unification\"\n)\n\nlazy val scalac213Options = {\n val exclusions = Set(\n \"-Yno-adapted-args\",\n \"-Ywarn-unused-import\",\n \"-Xfuture\",\n // type TraversableOnce in package scala is deprecated, symbol literal is deprecated; use Symbol(\"a\") instead\n \"-Xfatal-warnings\",\n \"-Ypartial-unification\"\n )\n\n // https://github.com/scala/bug/issues/12072\n val options = Seq(\"-Xlint:-byname-implicit\")\n scalac212Options.filter(s => !exclusions.contains(s)) ++ options\n}\n\nlazy val scalacOptionSettings = Def.setting {\n def baseScalacOptions(scalaVersion: String) =\n CrossVersion.partialVersion(scalaVersion) match {\n case Some((2, 13)) => scalac213Options\n case _ => scalac212Options\n }\n\n baseScalacOptions(scalaVersion.value)\n}\n\nlazy val framelessSettings = Seq(\n scalacOptions ++= scalacOptionSettings.value,\n Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, \"-oDF\"),\n libraryDependencies ++= Seq(\n \"com.chuusai\" %% \"shapeless\" % shapeless,\n \"org.scalatest\" %% \"scalatest\" % scalatest % Test,\n \"org.scalatestplus\" %% \"scalatestplus-scalacheck\" % scalatestplus % Test,\n \"org.scalacheck\" %% \"scalacheck\" % scalacheck % Test\n ),\n Test / javaOptions ++= {\n val baseOptions = Seq(\"-Xmx1G\", \"-ea\")\n val java17Options =\n if (sys.props(\"java.specification.version\").toDouble >= 17.0) {\n Seq(\n \"--add-opens=java.base/java.lang=ALL-UNNAMED\",\n \"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED\",\n \"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED\",\n \"--add-opens=java.base/java.io=ALL-UNNAMED\",\n \"--add-opens=java.base/java.net=ALL-UNNAMED\",\n \"--add-opens=java.base/java.nio=ALL-UNNAMED\",\n \"--add-opens=java.base/java.util=ALL-UNNAMED\",\n \"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED\",\n \"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED\",\n \"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED\",\n \"--add-opens=java.base/sun.nio.cs=ALL-UNNAMED\",\n \"--add-opens=java.base/sun.security.action=ALL-UNNAMED\",\n \"--add-opens=java.base/sun.util.calendar=ALL-UNNAMED\"\n )\n } else Seq.empty\n baseOptions ++ java17Options\n },\n Test / fork := true,\n Test / parallelExecution := false,\n mimaPreviousArtifacts ~= {\n _.filterNot(_.revision == \"0.11.0\") // didn't release properly\n },\n /**\n * The old Scala XML is pulled from Scala 2.12.x.\n *\n * [error] (update) found version conflict(s) in library dependencies; some are suspected to be binary incompatible:\n * [error]\n * [error] \t* org.scala-lang.modules:scala-xml_2.12:2.3.0 (early-semver) is selected over 1.0.6\n * [error] \t +- org.scoverage:scalac-scoverage-reporter_2.12:2.0.7 (depends on 2.4.0)\n * [error] \t +- org.scala-lang:scala-compiler:2.12.16 (depends on 1.0.6)\n */\n libraryDependencySchemes += \"org.scala-lang.modules\" %% \"scala-xml\" % VersionScheme.Always\n) ++ consoleSettings\n\nlazy val spark34Settings = Seq[Setting[_]](\n tlVersionIntroduced := Map(\"2.12\" -> \"0.14.1\", \"2.13\" -> \"0.14.1\"),\n mimaPreviousArtifacts := Set(\n organization.value %% moduleName.value\n .split(\"-\")\n .dropRight(1)\n .mkString(\"-\") % \"0.14.1\"\n )\n)\n\nlazy val spark33Settings = Seq[Setting[_]](\n tlVersionIntroduced := Map(\"2.12\" -> \"0.13.0\", \"2.13\" -> \"0.13.0\"),\n mimaPreviousArtifacts := Set(\n organization.value %% moduleName.value\n .split(\"-\")\n .dropRight(1)\n .mkString(\"-\") % \"0.14.0\"\n )\n)\n\nlazy val consoleSettings = Seq(\n Compile / console / scalacOptions ~= {\n _.filterNot(\"-Ywarn-unused-import\" == _)\n },\n Test / console / scalacOptions := (Compile / console / scalacOptions).value\n)\n\nlazy val framelessTypedDatasetREPL = Seq(\n initialize ~= { _ => // Color REPL\n val ansi = System.getProperty(\"sbt.log.noformat\", \"false\") != \"true\"\n if (ansi) System.setProperty(\"scala.color\", \"true\")\n },\n console / initialCommands :=\n \"\"\"\n |import org.apache.spark.{SparkConf, SparkContext}\n |import org.apache.spark.sql.SparkSession\n |import frameless.functions.aggregate._\n |import frameless.syntax._\n |\n |val conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"frameless repl\").set(\"spark.ui.enabled\", \"false\")\n |implicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\n |\n |import spark.implicits._\n |\n |spark.sparkContext.setLogLevel(\"WARN\")\n |\n |import frameless.TypedDataset\n \"\"\".stripMargin,\n console / cleanupCommands :=\n \"\"\"\n |spark.stop()\n \"\"\".stripMargin\n)\n\nThisBuild / organization := \"org.typelevel\"\nThisBuild / licenses := List(\n \"Apache-2.0\" -> url(\"http://opensource.org/licenses/Apache-2.0\")\n)\nThisBuild / developers := List(\n \"OlivierBlanvillain\" -> \"Olivier Blanvillain\",\n \"adelbertc\" -> \"Adelbert Chang\",\n \"imarios\" -> \"Marios Iliofotou\",\n \"kanterov\" -> \"Gleb Kanterov\",\n \"non\" -> \"Erik Osheim\",\n \"jeremyrsmith\" -> \"Jeremy Smith\",\n \"cchantep\" -> \"Cédric Chantepie\",\n \"pomadchin\" -> \"Grigory Pomadchin\"\n).map {\n case (username, fullName) =>\n tlGitHubDev(username, fullName)\n}\n\nThisBuild / tlCiReleaseBranches := Seq(\"master\")\nThisBuild / tlSitePublishBranch := Some(\"master\")\n\nval roots = List(\"root-spark33\", \"root-spark34\", \"root-spark35\")\n\nThisBuild / githubWorkflowBuildMatrixAdditions += \"project\" -> roots\n\nThisBuild / githubWorkflowBuildMatrixExclusions ++= roots.init.map { project =>\n MatrixExclude(Map(\"scala\" -> \"2.13\", \"project\" -> project))\n}\n\nThisBuild / githubWorkflowEnv += \"SBT_OPTS\" -> \"-Xms1g -Xmx4g\"\n" }, { "path": "cats/src/main/scala/frameless/cats/FramelessSyntax.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats.effect.Sync\nimport _root_.cats.syntax.all._\nimport _root_.cats.mtl.Ask\nimport org.apache.spark.sql.SparkSession\n\ntrait FramelessSyntax extends frameless.FramelessSyntax {\n implicit class SparkJobOps[F[_], A](fa: F[A])(implicit S: Sync[F], A: Ask[F, SparkSession]) {\n import S._, A._\n\n def withLocalProperty(key: String, value: String): F[A] =\n for {\n session <- ask\n _ <- delay(session.sparkContext.setLocalProperty(key, value))\n a <- fa\n } yield a\n\n def withGroupId(groupId: String): F[A] = withLocalProperty(\"spark.jobGroup.id\", groupId)\n\n def withDescription(description: String): F[A] = withLocalProperty(\"spark.job.description\", description)\n }\n}\n" }, { "path": "cats/src/main/scala/frameless/cats/SparkDelayInstances.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats.effect.Sync\nimport org.apache.spark.sql.SparkSession\n\ntrait SparkDelayInstances {\n implicit def framelessCatsSparkDelayForSync[F[_]](implicit S: Sync[F]): SparkDelay[F] = new SparkDelay[F] {\n def delay[A](a: => A)(implicit spark: SparkSession): F[A] = S.delay(a)\n }\n}\n" }, { "path": "cats/src/main/scala/frameless/cats/SparkTask.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats.Id\nimport _root_.cats.data.Kleisli\nimport org.apache.spark.SparkContext\n\nobject SparkTask {\n def apply[A](f: SparkContext => A): SparkTask[A] =\n Kleisli[Id, SparkContext, A](f)\n\n def pure[A](a: => A): SparkTask[A] =\n Kleisli[Id, SparkContext, A](_ => a)\n}\n" }, { "path": "cats/src/main/scala/frameless/cats/implicits.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats._\nimport _root_.cats.kernel.{CommutativeMonoid, CommutativeSemigroup}\nimport _root_.cats.syntax.all._\nimport alleycats.Empty\n\nimport scala.reflect.ClassTag\nimport org.apache.spark.rdd.RDD\n\nobject implicits extends FramelessSyntax with SparkDelayInstances {\n implicit class rddOps[A: ClassTag](lhs: RDD[A]) {\n def csum(implicit m: CommutativeMonoid[A]): A =\n lhs.fold(m.empty)(_ |+| _)\n def csumOption(implicit m: CommutativeSemigroup[A]): Option[A] =\n lhs.aggregate[Option[A]](None)(\n (acc, a) => Some(acc.fold(a)(_ |+| a)),\n (l, r) => l.fold(r)(x => r.map(_ |+| x) orElse Some(x))\n )\n\n def cmin(implicit o: Order[A], e: Empty[A]): A = {\n if (lhs.isEmpty()) e.empty\n else lhs.reduce(_ min _)\n }\n def cminOption(implicit o: Order[A]): Option[A] =\n csumOption(new CommutativeSemigroup[A] {\n def combine(l: A, r: A) = l min r\n })\n\n def cmax(implicit o: Order[A], e: Empty[A]): A = {\n if (lhs.isEmpty()) e.empty\n else lhs.reduce(_ max _)\n }\n def cmaxOption(implicit o: Order[A]): Option[A] =\n csumOption(new CommutativeSemigroup[A] {\n def combine(l: A, r: A) = l max r\n })\n }\n\n implicit class pairRddOps[K: ClassTag, V: ClassTag](lhs: RDD[(K, V)]) {\n def csumByKey(implicit m: CommutativeSemigroup[V]): RDD[(K, V)] = lhs.reduceByKey(_ |+| _)\n def cminByKey(implicit o: Order[V]): RDD[(K, V)] = lhs.reduceByKey(_ min _)\n def cmaxByKey(implicit o: Order[V]): RDD[(K, V)] = lhs.reduceByKey(_ max _)\n }\n}\n\nobject union {\n implicit def unionSemigroup[A]: Semigroup[RDD[A]] =\n new Semigroup[RDD[A]] {\n def combine(lhs: RDD[A], rhs: RDD[A]): RDD[A] = lhs union rhs\n }\n}\n\nobject inner {\n implicit def pairwiseInnerSemigroup[K: ClassTag, V: ClassTag: Semigroup]: Semigroup[RDD[(K, V)]] =\n new Semigroup[RDD[(K, V)]] {\n def combine(lhs: RDD[(K, V)], rhs: RDD[(K, V)]): RDD[(K, V)] =\n lhs.join(rhs).mapValues { case (x, y) => x |+| y }\n }\n}\n\nobject outer {\n implicit def pairwiseOuterSemigroup[K: ClassTag, V: ClassTag](implicit m: Monoid[V]): Semigroup[RDD[(K, V)]] =\n new Semigroup[RDD[(K, V)]] {\n def combine(lhs: RDD[(K, V)], rhs: RDD[(K, V)]): RDD[(K, V)] =\n lhs.fullOuterJoin(rhs).mapValues {\n case (Some(x), Some(y)) => x |+| y\n case (None, Some(y)) => y\n case (Some(x), None) => x\n case (None, None) => m.empty\n }\n }\n}\n" }, { "path": "cats/src/main/scala/frameless/cats/package.scala", "content": "package frameless\n\nimport _root_.cats.Id\nimport _root_.cats.data.Kleisli\nimport org.apache.spark.SparkContext\n\npackage object cats {\n type SparkTask[A] = Kleisli[Id, SparkContext, A]\n}\n" }, { "path": "cats/src/test/resources/log4j.properties", "content": "log4j.logger.akka.event.slf4j.Slf4jLogger=ERROR\nlog4j.logger.akka.event.slf4j=ERROR\nlog4j.logger.akka.remote.EndpointWriter=ERROR\nlog4j.logger.akka.remote.RemoteActorRefProvider$RemotingTerminator=ERROR\nlog4j.logger.com.anjuke.dm=ERROR\nlog4j.logger.io.netty.bootstrap.ServerBootstrap=ERROR\nlog4j.logger.io.netty.buffer.ByteBufUtil=ERROR\nlog4j.logger.io.netty.buffer.PooledByteBufAllocator=ERROR\nlog4j.logger.io.netty.channel.AbstractChannel=ERROR\nlog4j.logger.io.netty.channel.ChannelInitializer=ERROR\nlog4j.logger.io.netty.channel.ChannelOutboundBuffer=ERROR\nlog4j.logger.io.netty.channel.DefaultChannelPipeline=ERROR\nlog4j.logger.io.netty.channel.MultithreadEventLoopGroup=ERROR\nlog4j.logger.io.netty.channel.nio.AbstractNioChannel=ERROR\nlog4j.logger.io.netty.channel.nio.NioEventLoop=ERROR\nlog4j.logger.io.netty.channel.socket.nio.NioServerSocketChannel=ERROR\nlog4j.logger.io.netty.util.concurrent.DefaultPromise.rejectedExecution=ERROR\nlog4j.logger.io.netty.util.concurrent.DefaultPromise=ERROR\nlog4j.logger.io.netty.util.concurrent.GlobalEventExecutor=ERROR\nlog4j.logger.io.netty.util.concurrent.SingleThreadEventExecutor=ERROR\nlog4j.logger.io.netty.util.internal.logging.InternalLoggerFactory=ERROR\nlog4j.logger.io.netty.util.internal.PlatformDependent0=ERROR\nlog4j.logger.io.netty.util.internal.PlatformDependent=ERROR\nlog4j.logger.io.netty.util.internal.SystemPropertyUtil=ERROR\nlog4j.logger.io.netty.util.internal.ThreadLocalRandom=ERROR\nlog4j.logger.io.netty.util.NetUtil=ERROR\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=ERROR\nlog4j.logger.org.apache.hadoop.conf.Configuration=ERROR\nlog4j.logger.org.apache.hadoop.fs.FileSystem=ERROR\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=ERROR\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\nlog4j.logger.org.apache.hadoop.mapred.JobConf=ERROR\nlog4j.logger.org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.Interns=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.MetricsSourceBuilder=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.MutableMetricsFactory=ERROR\nlog4j.logger.org.apache.hadoop.security.authentication.util.KerberosName=ERROR\nlog4j.logger.org.apache.hadoop.security.Groups=ERROR\nlog4j.logger.org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback=ERROR\nlog4j.logger.org.apache.hadoop.security.SecurityUtil=ERROR\nlog4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR\nlog4j.logger.org.apache.hadoop.security.UserGroupInformation=ERROR\nlog4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR\nlog4j.logger.org.apache.hadoop.util.ShutdownHookManager=ERROR\nlog4j.logger.org.apache.spark.broadcast.TorrentBroadcast=ERROR\nlog4j.logger.org.apache.spark.ContextCleaner=ERROR\nlog4j.logger.org.apache.spark.executor.Executor=ERROR\nlog4j.logger.org.apache.spark.HeartbeatReceiver=ERROR\nlog4j.logger.org.apache.spark.HttpFileServer=ERROR\nlog4j.logger.org.apache.spark.HttpServer=ERROR\nlog4j.logger.org.apache.spark.MapOutputTrackerMaster=ERROR\nlog4j.logger.org.apache.spark.MapOutputTrackerMasterEndpoint=ERROR\nlog4j.logger.org.apache.spark.metrics.MetricsSystem=ERROR\nlog4j.logger.org.apache.spark.network.client.TransportClientFactory=ERROR\nlog4j.logger.org.apache.spark.network.netty.NettyBlockTransferService=ERROR\nlog4j.logger.org.apache.spark.network.protocol.MessageDecoder=ERROR\nlog4j.logger.org.apache.spark.network.protocol.MessageEncoder=ERROR\nlog4j.logger.org.apache.spark.network.server.OneForOneStreamManager=ERROR\nlog4j.logger.org.apache.spark.network.server.TransportServer=ERROR\nlog4j.logger.org.apache.spark.network.TransportContext=ERROR\nlog4j.logger.org.apache.spark.network.util.JavaUtils=ERROR\nlog4j.logger.org.apache.spark.rdd.CoGroupedRDD=ERROR\nlog4j.logger.org.apache.spark.rdd.SubtractedRDD=ERROR\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR\nlog4j.logger.org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1=ERROR\nlog4j.logger.org.apache.spark.scheduler.DAGScheduler=ERROR\nlog4j.logger.org.apache.spark.scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint=ERROR\nlog4j.logger.org.apache.spark.scheduler.TaskSchedulerImpl=ERROR\nlog4j.logger.org.apache.spark.scheduler.TaskSetManager=ERROR\nlog4j.logger.org.apache.spark.SecurityManager=ERROR\nlog4j.logger.org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter=ERROR\nlog4j.logger.org.apache.spark.SparkContext=ERROR\nlog4j.logger.org.apache.spark.SparkEnv=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GeneratePredicate=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeRowJoiner=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.package$ExpressionCanonicalizer=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.optimizer.DefaultOptimizer=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys=ERROR\nlog4j.logger.org.apache.spark.sql.execution.aggregate.SortBasedAggregate=ERROR\nlog4j.logger.org.apache.spark.sql.execution.aggregate.TungstenAggregate=ERROR\nlog4j.logger.org.apache.spark.sql.execution.Exchange=ERROR\nlog4j.logger.org.apache.spark.sql.execution.joins.ShuffledHashOuterJoin=ERROR\nlog4j.logger.org.apache.spark.sql.SQLContext$$anon$1=ERROR\nlog4j.logger.org.apache.spark.sql.SQLContext$$anon$2=ERROR\nlog4j.logger.org.apache.spark.SSLOptions=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManager=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerInfo=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerMaster=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerMasterEndpoint=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerSlaveEndpoint=ERROR\nlog4j.logger.org.apache.spark.storage.DiskBlockManager=ERROR\nlog4j.logger.org.apache.spark.storage.MemoryStore=ERROR\nlog4j.logger.org.apache.spark.storage.ShuffleBlockFetcherIterator=ERROR\nlog4j.logger.org.apache.spark.ui.SparkUI=ERROR\nlog4j.logger.org.apache.spark.unsafe.map.BytesToBytesMap=ERROR\nlog4j.logger.org.apache.spark.unsafe.memory.TaskMemoryManager=ERROR\nlog4j.logger.org.apache.spark.util.AkkaUtils=ERROR\nlog4j.logger.org.apache.spark.util.ClosureCleaner=ERROR\nlog4j.logger.org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter=ERROR\nlog4j.logger.org.apache.spark.util.Utils=ERROR\nlog4j.logger.org.apache.spark=ERROR\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.eclipse.jetty=ERROR\nlog4j.logger.org.spark-project.jetty.http.AbstractGenerator=ERROR\nlog4j.logger.org.spark-project.jetty.http.HttpGenerator=ERROR\nlog4j.logger.org.spark-project.jetty.http.MimeTypes=ERROR\nlog4j.logger.org.spark-project.jetty.io.AbstractBuffer=ERROR\nlog4j.logger.org.spark-project.jetty.io.nio=ERROR\nlog4j.logger.org.spark-project.jetty.server.AbstractConnector=ERROR\nlog4j.logger.org.spark-project.jetty.server.bio.SocketConnector=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.AbstractHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ContextHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ContextHandlerCollection=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.DefaultHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ErrorHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.GzipHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ResourceHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.Server=ERROR\nlog4j.logger.org.spark-project.jetty.server=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.DefaultServlet=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.Holder=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.ServletHandler=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.ServletHolder=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.AggregateLifeCycle=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.Container=ERROR\nlog4j.logger.org.spark-project.jetty.util.IO=ERROR\nlog4j.logger.org.spark-project.jetty.util.log=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.FileResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.JarFileResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.JarResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.Resource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.URLResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.StringUtil=ERROR\nlog4j.logger.org.spark-project.jetty.util.thread.QueuedThreadPool=ERROR\nlog4j.logger.org.spark-project.jetty.util.thread.Timeout=ERROR\nlog4j.logger.org.spark-project.jetty=ERROR\nlog4j.logger.Remoting=ERROR\n" }, { "path": "cats/src/test/resources/log4j2.properties", "content": "# Set to debug or trace if log4j initialization is failing\nstatus = warn\n\n# Name of the configuration\nname = ConsoleAppender\n\n# Console appender configuration\nappender.console.type = Console\nappender.console.name = consoleLogger\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{YYYY-MM-dd HH:mm:ss} [%t] %-5p %c:%L - %m%n\nappender.console.target = SYSTEM_OUT\n\n# Root logger level\nrootLogger.level = error\n\n# Root logger referring to console appender\nrootLogger.appenderRef.stdout.ref = consoleLogger\n\nlogger.spark.name = org.apache.spark\nlogger.spark.level = warn\n\nlogger.hadoop.name = org.apache.hadoop\nlogger.hadoop.level = warn\n" }, { "path": "cats/src/test/scala/frameless/cats/FramelessSyntaxTests.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats.data.ReaderT\nimport _root_.cats.effect.IO\nimport _root_.cats.effect.unsafe.implicits.global\nimport org.apache.spark.sql.SparkSession\nimport org.scalatest.matchers.should.Matchers\nimport org.scalacheck.{Test => PTest}\nimport org.scalacheck.Prop, Prop._\nimport org.scalacheck.effect.PropF, PropF._\n\nclass FramelessSyntaxTests extends TypedDatasetSuite with Matchers {\n override val sparkDelay = null\n\n def prop[A, B](data: Vector[X2[A, B]])(\n implicit ev: TypedEncoder[X2[A, B]]\n ): Prop = {\n import implicits._\n\n val dataset = TypedDataset.create(data).dataset\n val dataframe = dataset.toDF()\n\n val typedDataset = dataset.typed\n val typedDatasetFromDataFrame = dataframe.unsafeTyped[X2[A, B]]\n\n typedDataset.collect[IO]().unsafeRunSync().toVector ?= typedDatasetFromDataFrame.collect[IO]().unsafeRunSync().toVector\n }\n\n test(\"dataset typed - toTyped\") {\n check(forAll(prop[Int, String] _))\n }\n\n test(\"properties can be read back\") {\n import implicits._\n import _root_.cats.syntax.all._\n\n forAllF { (k: String, v: String) =>\n val scopedKey = \"frameless.tests.\" + k\n 1\n .pure[ReaderT[IO, SparkSession, *]]\n .withLocalProperty(scopedKey, v)\n .withGroupId(v)\n .withDescription(v)\n .run(session)\n .map { _ =>\n sc.getLocalProperty(scopedKey) shouldBe v\n sc.getLocalProperty(\"spark.jobGroup.id\") shouldBe v\n sc.getLocalProperty(\"spark.job.description\") shouldBe v\n }.void\n }.check().unsafeRunSync().status shouldBe PTest.Passed\n }\n}\n" }, { "path": "cats/src/test/scala/frameless/cats/test.scala", "content": "package frameless\npackage cats\n\nimport _root_.cats.Foldable\nimport _root_.cats.syntax.all._\n\nimport org.apache.spark.SparkContext\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.rdd.RDD\nimport org.apache.spark.{SparkConf, SparkContext => SC}\n\nimport org.scalatest.compatible.Assertion\nimport org.scalactic.anyvals.PosInt\nimport org.scalacheck.Arbitrary\nimport org.scalatestplus.scalacheck.ScalaCheckPropertyChecks\nimport Arbitrary._\n\nimport scala.collection.immutable.SortedMap\nimport scala.reflect.ClassTag\nimport org.scalatest.matchers.should.Matchers\nimport org.scalatest.propspec.AnyPropSpec\n\ntrait SparkTests {\n val appID: String = new java.util.Date().toString + math.floor(math.random() * 10E4).toLong.toString\n\n val conf: SparkConf = new SparkConf()\n .setMaster(\"local[*]\")\n .setAppName(\"test\")\n .set(\"spark.ui.enabled\", \"false\")\n .set(\"spark.app.id\", appID)\n\n implicit def session: SparkSession = SparkSession.builder().config(conf).getOrCreate()\n implicit def sc: SparkContext = session.sparkContext\n\n implicit class seqToRdd[A: ClassTag](seq: Seq[A])(implicit sc: SC) {\n def toRdd: RDD[A] = sc.makeRDD(seq)\n }\n}\n\nobject Tests {\n def innerPairwise(mx: Map[String, Int], my: Map[String, Int], check: (Any, Any) => Assertion)(implicit sc: SC): Assertion = {\n import frameless.cats.implicits._\n import frameless.cats.inner._\n val xs = sc.parallelize(mx.toSeq)\n val ys = sc.parallelize(my.toSeq)\n\n val mz0 = (xs |+| ys).collectAsMap()\n val mz1 = (xs join ys).mapValues { case (x, y) => x |+| y }.collectAsMap()\n val mz2 = (for { (k, x) <- mx; y <- my.get(k) } yield (k, x + y)).toMap\n check(mz0, mz1)\n check(mz1, mz2)\n\n val zs = sc.parallelize(mx.values.toSeq)\n check(xs.csumByKey.collectAsMap(), mx)\n check(zs.csum, zs.collect().sum)\n\n if (mx.nonEmpty) {\n check(xs.cminByKey.collectAsMap(), mx)\n check(xs.cmaxByKey.collectAsMap(), mx)\n check(zs.cmin, zs.collect().min)\n check(zs.cmax, zs.collect().max)\n } else check(1, 1)\n }\n}\n\nclass Test extends AnyPropSpec with Matchers with ScalaCheckPropertyChecks with SparkTests {\n implicit override val generatorDrivenConfig =\n PropertyCheckConfiguration(minSize = PosInt(10))\n\n property(\"spark is working\") {\n sc.parallelize(Seq(1, 2, 3)).collect() shouldBe Array(1,2,3)\n }\n\n property(\"inner pairwise monoid\") {\n // Make sure we have non-empty map\n forAll { (xh: (String, Int), mx: Map[String, Int], yh: (String, Int), my: Map[String, Int]) =>\n Tests.innerPairwise(mx + xh, my + yh, _ shouldBe _)\n }\n }\n\n property(\"rdd simple numeric commutative semigroup\") {\n import frameless.cats.implicits._\n\n forAll { seq: List[Int] =>\n val expectedSum = if (seq.isEmpty) None else Some(seq.sum)\n val expectedMin = if (seq.isEmpty) None else Some(seq.min)\n val expectedMax = if (seq.isEmpty) None else Some(seq.max)\n\n val rdd = seq.toRdd\n\n rdd.cmin shouldBe expectedMin.getOrElse(0)\n rdd.cminOption shouldBe expectedMin\n\n rdd.cmax shouldBe expectedMax.getOrElse(0)\n rdd.cmaxOption shouldBe expectedMax\n\n rdd.csum shouldBe expectedSum.getOrElse(0)\n rdd.csumOption shouldBe expectedSum\n }\n }\n\n property(\"rdd of SortedMap[Int,Int] commutative monoid\") {\n import frameless.cats.implicits._\n forAll { seq: List[SortedMap[Int, Int]] =>\n val rdd = seq.toRdd\n rdd.csum shouldBe Foldable[List].fold(seq)\n }\n }\n\n property(\"rdd tuple commutative semigroup example\") {\n import frameless.cats.implicits._\n forAll { seq: List[(Int, Int)] =>\n val expectedSum = if (seq.isEmpty) None else Some(Foldable[List].fold(seq))\n val rdd = seq.toRdd\n\n rdd.csum shouldBe expectedSum.getOrElse(0 -> 0)\n rdd.csumOption shouldBe expectedSum\n }\n }\n\n property(\"pair rdd numeric commutative semigroup example\") {\n import frameless.cats.implicits._\n val seq = Seq( (\"a\",2), (\"b\",3), (\"d\",6), (\"b\",2), (\"d\",1) )\n val rdd = seq.toRdd\n rdd.cminByKey.collect().toSeq should contain theSameElementsAs Seq( (\"a\",2), (\"b\",2), (\"d\",1) )\n rdd.cmaxByKey.collect().toSeq should contain theSameElementsAs Seq( (\"a\",2), (\"b\",3), (\"d\",6) )\n rdd.csumByKey.collect().toSeq should contain theSameElementsAs Seq( (\"a\",2), (\"b\",5), (\"d\",7) )\n }\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystAverageable.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/**\n * When averaging Spark doesn't change these types:\n * - BigDecimal -> BigDecimal\n * - Double -> Double\n * But it changes these types :\n * - Int -> Double\n * - Short -> Double\n * - Long -> Double\n */\n@implicitNotFound(\"Cannot compute average of type ${In}.\")\ntrait CatalystAverageable[In, Out]\n\nobject CatalystAverageable {\n private[this] val theInstance = new CatalystAverageable[Any, Any] {}\n private[this] def of[In, Out]: CatalystAverageable[In, Out] = theInstance.asInstanceOf[CatalystAverageable[In, Out]]\n\n implicit val framelessAverageableBigDecimal: CatalystAverageable[BigDecimal, BigDecimal] = of[BigDecimal, BigDecimal]\n implicit val framelessAverageableDouble: CatalystAverageable[Double, Double] = of[Double, Double]\n implicit val framelessAverageableLong: CatalystAverageable[Long, Double] = of[Long, Double]\n implicit val framelessAverageableInt: CatalystAverageable[Int, Double] = of[Int, Double]\n implicit val framelessAverageableShort: CatalystAverageable[Short, Double] = of[Short, Double]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystBitShift.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Spark does not return always Int on shift\n */\n\n@implicitNotFound(\"Cannot do bit shift operations on columns of type ${In}.\")\ntrait CatalystBitShift[In, Out]\n\nobject CatalystBitShift {\n private[this] val theInstance = new CatalystBitShift[Any, Any] {}\n private[this] def of[In, Out]: CatalystBitShift[In, Out] = theInstance.asInstanceOf[CatalystBitShift[In, Out]]\n\n implicit val framelessBitShiftBigDecimal: CatalystBitShift[BigDecimal, Int] = of[BigDecimal, Int]\n implicit val framelessBitShiftDouble : CatalystBitShift[Byte, Int] = of[Byte, Int]\n implicit val framelessBitShiftInt : CatalystBitShift[Short, Int] = of[Short, Int]\n implicit val framelessBitShiftLong : CatalystBitShift[Int, Int] = of[Int, Int]\n implicit val framelessBitShiftShort : CatalystBitShift[Long, Long] = of[Long, Long]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystBitwise.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/**\n * Types that can be bitwise ORed, ANDed, or XORed by Catalyst.\n * Note that Catalyst requires that when performing bitwise operations between columns\n * the two types must be the same so in some cases casting is necessary.\n */\n@implicitNotFound(\"Cannot do bitwise operations on columns of type ${A}.\")\ntrait CatalystBitwise[A] extends CatalystNumeric[A]\n\nobject CatalystBitwise {\n private[this] val theInstance = new CatalystBitwise[Any] {}\n\n private[this] def of[A]: CatalystBitwise[A] =\n theInstance.asInstanceOf[CatalystBitwise[A]]\n\n implicit val framelessbyteBitwise: CatalystBitwise[Byte] = of[Byte]\n implicit val framelessshortBitwise: CatalystBitwise[Short] = of[Short]\n implicit val framelessintBitwise: CatalystBitwise[Int] = of[Int]\n implicit val framelesslongBitwise: CatalystBitwise[Long] = of[Long]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystCast.scala", "content": "package frameless\n\ntrait CatalystCast[A, B]\n\nobject CatalystCast {\n private[this] val theInstance = new CatalystCast[Any, Any] {}\n private[this] def of[A, B]: CatalystCast[A, B] = theInstance.asInstanceOf[CatalystCast[A, B]]\n\n implicit def framelessCastToString[T]: CatalystCast[T, String] = of[T, String]\n\n implicit def framelessNumericToLong [A: CatalystNumeric]: CatalystCast[A, Long] = of[A, Long]\n implicit def framelessNumericToInt [A: CatalystNumeric]: CatalystCast[A, Int] = of[A, Int]\n implicit def framelessNumericToShort [A: CatalystNumeric]: CatalystCast[A, Short] = of[A, Short]\n implicit def framelessNumericToByte [A: CatalystNumeric]: CatalystCast[A, Byte] = of[A, Byte]\n implicit def framelessNumericToDecimal[A: CatalystNumeric]: CatalystCast[A, BigDecimal] = of[A, BigDecimal]\n implicit def framelessNumericToDouble [A: CatalystNumeric]: CatalystCast[A, Double] = of[A, Double]\n\n implicit def framelessBooleanToNumeric[A: CatalystNumeric]: CatalystCast[Boolean, A] = of[Boolean, A]\n\n // doesn't make any sense to include:\n // - sqlDateToBoolean: always None\n // - sqlTimestampToBoolean: compares us to 0\n implicit val framelessStringToBoolean : CatalystCast[String, Option[Boolean]] = of[String, Option[Boolean]]\n implicit val framelessLongToBoolean : CatalystCast[Long, Boolean] = of[Long, Boolean]\n implicit val framelessIntToBoolean : CatalystCast[Int, Boolean] = of[Int, Boolean]\n implicit val framelessShortToBoolean : CatalystCast[Short, Boolean] = of[Short, Boolean]\n implicit val framelessByteToBoolean : CatalystCast[Byte, Boolean] = of[Byte, Boolean]\n implicit val framelessBigDecimalToBoolean: CatalystCast[BigDecimal, Boolean] = of[BigDecimal, Boolean]\n implicit val framelessDoubleToBoolean : CatalystCast[Double, Boolean] = of[Double, Boolean]\n\n // TODO\n\n // needs verification, does it make sense to include? probably better as a separate function\n // implicit object stringToInt extends CatalystCast[String, Option[Int]]\n // implicit object stringToShort extends CatalystCast[String, Option[Short]]\n // implicit object stringToByte extends CatalystCast[String, Option[Byte]]\n // implicit object stringToDecimal extends CatalystCast[String, Option[BigDecimal]]\n // implicit object stringToLong extends CatalystCast[String, Option[Long]]\n // implicit object stringToSqlDate extends CatalystCast[String, Option[SQLDate]]\n\n\n // needs verification:\n //implicit object sqlTimestampToSqlDate extends CatalystCast[SQLTimestamp, SQLDate]\n\n // needs verification:\n // implicit object sqlTimestampToDecimal extends CatalystCast[SQLTimestamp, BigDecimal]\n // implicit object sqlTimestampToLong extends CatalystCast[SQLTimestamp, Long]\n\n // needs verification:\n // implicit object stringToSqlTimestamp extends CatalystCast[String, SQLTimestamp]\n // implicit object longToSqlTimestamp extends CatalystCast[Long, SQLTimestamp]\n // implicit object intToSqlTimestamp extends CatalystCast[Int, SQLTimestamp]\n // implicit object doubleToSqlTimestamp extends CatalystCast[Double, SQLTimestamp]\n // implicit object floatToSqlTimestamp extends CatalystCast[Float, SQLTimestamp]\n // implicit object bigDecimalToSqlTimestamp extends CatalystCast[BigDecimal, SQLTimestamp]\n // implicit object sqlDateToSqlTimestamp extends CatalystCast[SQLDate, SQLTimestamp]\n\n // doesn't make sense to include:\n // - booleanToSqlTimestamp: 1L or 0L\n // - shortToSqlTimestamp: ???\n // - byteToSqlTimestamp: ???\n\n // doesn't make sense to include:\n // - sqlDateToLong: always None\n // - sqlDateToInt: always None\n // - sqlDateToInt: always None\n // - sqlDateToInt: always None\n // - sqlDateToInt: always None\n\n // doesn't make sense to include:\n // - sqlTimestampToInt: useful? can be done through `-> Long -> Int`\n // - sqlTimestampToShort: useful? can be done through `-> Long -> Int`\n // - sqlTimestampToShort: useful? can be done through `-> Long -> Int`\n\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystCollection.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n@implicitNotFound(\"Cannot do collection operations on columns of type ${C}.\")\ntrait CatalystCollection[C[_]]\n\nobject CatalystCollection {\n private[this] val theInstance = new CatalystCollection[Any] {}\n private[this] def of[A[_]]: CatalystCollection[A] = theInstance.asInstanceOf[CatalystCollection[A]]\n\n implicit val arrayObject : CatalystCollection[Array] = of[Array]\n implicit val seqObject : CatalystCollection[Seq] = of[Seq]\n implicit val listObject : CatalystCollection[List] = of[List]\n implicit val vectorObject: CatalystCollection[Vector] = of[Vector]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystDivisible.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Spark divides everything as Double, expect BigDecimals are divided into\n * another BigDecimal, benefiting from some added precision.\n */\n@implicitNotFound(\"Cannot compute division on type ${In}.\")\ntrait CatalystDivisible[In, Out]\n\nobject CatalystDivisible {\n private[this] val theInstance = new CatalystDivisible[Any, Any] {}\n private[this] def of[In, Out]: CatalystDivisible[In, Out] = theInstance.asInstanceOf[CatalystDivisible[In, Out]]\n\n implicit val framelessDivisibleBigDecimal: CatalystDivisible[BigDecimal, BigDecimal] = of[BigDecimal, BigDecimal]\n implicit val framelessDivisibleDouble : CatalystDivisible[Double, Double] = of[Double, Double]\n implicit val framelessDivisibleInt : CatalystDivisible[Int, Double] = of[Int, Double]\n implicit val framelessDivisibleLong : CatalystDivisible[Long, Double] = of[Long, Double]\n implicit val framelessDivisibleByte : CatalystDivisible[Byte, Double] = of[Byte, Double]\n implicit val framelessDivisibleShort : CatalystDivisible[Short, Double] = of[Short, Double]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystIsin.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Types for which we can check if is in */\n@implicitNotFound(\"Cannot do isin operation on columns of type ${A}.\")\ntrait CatalystIsin[A]\n\nobject CatalystIsin {\n implicit object framelessBigDecimal extends CatalystIsin[BigDecimal]\n implicit object framelessByte extends CatalystIsin[Byte]\n implicit object framelessDouble extends CatalystIsin[Double]\n implicit object framelessFloat extends CatalystIsin[Float]\n implicit object framelessInt extends CatalystIsin[Int]\n implicit object framelessLong extends CatalystIsin[Long]\n implicit object framelessShort extends CatalystIsin[Short]\n implicit object framelesssString extends CatalystIsin[String]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystNaN.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Spark does NaN check only for these types */\n@implicitNotFound(\"Columns of type ${A} cannot be NaN.\")\ntrait CatalystNaN[A]\n\nobject CatalystNaN {\n private[this] val theInstance = new CatalystNaN[Any] {}\n private[this] def of[A]: CatalystNaN[A] = theInstance.asInstanceOf[CatalystNaN[A]]\n\n implicit val framelessFloatNaN : CatalystNaN[Float] = of[Float]\n implicit val framelessDoubleNaN : CatalystNaN[Double] = of[Double]\n}\n\n" }, { "path": "core/src/main/scala/frameless/CatalystNotNullable.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n@implicitNotFound(\"Cannot find evidence that type ${A} is nullable. Currently, only Option[A] is nullable.\")\ntrait CatalystNullable[A]\n\nobject CatalystNullable {\n implicit def optionIsNullable[A]: CatalystNullable[Option[A]] = new CatalystNullable[Option[A]] {}\n}\n\n@implicitNotFound(\"Cannot find evidence that type ${A} is not nullable.\")\ntrait NotCatalystNullable[A]\n\nobject NotCatalystNullable {\n implicit def everythingIsNotNullable[A]: NotCatalystNullable[A] = new NotCatalystNullable[A] {}\n implicit def nullableIsNotNotNullable[A: CatalystNullable]: NotCatalystNullable[A] = new NotCatalystNullable[A] {}\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystNumeric.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Types that can be added, subtracted and multiplied by Catalyst. */\n@implicitNotFound(\"Cannot do numeric operations on columns of type ${A}.\")\ntrait CatalystNumeric[A]\n\nobject CatalystNumeric {\n private[this] val theInstance = new CatalystNumeric[Any] {}\n private[this] def of[A]: CatalystNumeric[A] = theInstance.asInstanceOf[CatalystNumeric[A]]\n\n implicit val framelessbigDecimalNumeric: CatalystNumeric[BigDecimal] = of[BigDecimal]\n implicit val framelessbyteNumeric : CatalystNumeric[Byte] = of[Byte]\n implicit val framelessdoubleNumeric : CatalystNumeric[Double] = of[Double]\n implicit val framelessintNumeric : CatalystNumeric[Int] = of[Int]\n implicit val framelesslongNumeric : CatalystNumeric[Long] = of[Long]\n implicit val framelessshortNumeric : CatalystNumeric[Short] = of[Short]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystNumericWithJavaBigDecimal.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Spark does not return always the same type as the input was for example abs\n */\n@implicitNotFound(\"Cannot compute on type ${In}.\")\ntrait CatalystNumericWithJavaBigDecimal[In, Out]\n\nobject CatalystNumericWithJavaBigDecimal {\n private[this] val theInstance = new CatalystNumericWithJavaBigDecimal[Any, Any] {}\n private[this] def of[In, Out]: CatalystNumericWithJavaBigDecimal[In, Out] = theInstance.asInstanceOf[CatalystNumericWithJavaBigDecimal[In, Out]]\n\n implicit val framelessAbsoluteBigDecimal: CatalystNumericWithJavaBigDecimal[BigDecimal, java.math.BigDecimal] = of[BigDecimal, java.math.BigDecimal]\n implicit val framelessAbsoluteDouble : CatalystNumericWithJavaBigDecimal[Double, Double] = of[Double, Double]\n implicit val framelessAbsoluteInt : CatalystNumericWithJavaBigDecimal[Int, Int] = of[Int, Int]\n implicit val framelessAbsoluteLong : CatalystNumericWithJavaBigDecimal[Long, Long] = of[Long, Long]\n implicit val framelessAbsoluteShort : CatalystNumericWithJavaBigDecimal[Short, Short] = of[Short, Short]\n implicit val framelessAbsoluteByte : CatalystNumericWithJavaBigDecimal[Byte, Byte] = of[Byte, Byte]\n\n}" }, { "path": "core/src/main/scala/frameless/CatalystOrdered.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\nimport shapeless.{Generic, HList, Lazy}\nimport shapeless.ops.hlist.LiftAll\nimport java.time.{Duration, Instant, Period}\n\n/** Types that can be ordered/compared by Catalyst. */\n@implicitNotFound(\"Cannot compare columns of type ${A}.\")\ntrait CatalystOrdered[A]\n\nobject CatalystOrdered {\n private[this] val theInstance = new CatalystOrdered[Any] {}\n private[this] def of[A]: CatalystOrdered[A] = theInstance.asInstanceOf[CatalystOrdered[A]]\n\n implicit val framelessIntOrdered : CatalystOrdered[Int] = of[Int]\n implicit val framelessBooleanOrdered : CatalystOrdered[Boolean] = of[Boolean]\n implicit val framelessByteOrdered : CatalystOrdered[Byte] = of[Byte]\n implicit val framelessShortOrdered : CatalystOrdered[Short] = of[Short]\n implicit val framelessLongOrdered : CatalystOrdered[Long] = of[Long]\n implicit val framelessFloatOrdered : CatalystOrdered[Float] = of[Float]\n implicit val framelessDoubleOrdered : CatalystOrdered[Double] = of[Double]\n implicit val framelessBigDecimalOrdered : CatalystOrdered[BigDecimal] = of[BigDecimal]\n implicit val framelessSQLDateOrdered : CatalystOrdered[SQLDate] = of[SQLDate]\n implicit val framelessSQLTimestampOrdered: CatalystOrdered[SQLTimestamp] = of[SQLTimestamp]\n implicit val framelessStringOrdered : CatalystOrdered[String] = of[String]\n implicit val framelessInstantOrdered : CatalystOrdered[Instant] = of[Instant]\n implicit val framelessDurationOrdered : CatalystOrdered[Duration] = of[Duration]\n implicit val framelessPeriodOrdered : CatalystOrdered[Period] = of[Period]\n\n implicit def injectionOrdered[A, B]\n (implicit\n i0: Injection[A, B],\n i1: CatalystOrdered[B]\n ): CatalystOrdered[A] = of[A]\n\n implicit def deriveGeneric[G, H <: HList]\n (implicit\n i0: Generic.Aux[G, H],\n i1: Lazy[LiftAll[CatalystOrdered, H]]\n ): CatalystOrdered[G] = of[G]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystPivotable.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n@implicitNotFound(\"Cannot pivot on type ${A}. Currently supported types to pivot are {Int, Long, Boolean, and String}.\")\ntrait CatalystPivotable[A]\n\nobject CatalystPivotable {\n private[this] val theInstance = new CatalystPivotable[Any] {}\n private[this] def of[A]: CatalystPivotable[A] = theInstance.asInstanceOf[CatalystPivotable[A]]\n\n implicit val framelessIntPivotable : CatalystPivotable[Int] = of[Int]\n implicit val framelessLongPivotable : CatalystPivotable[Long] = of[Long]\n implicit val framelessBooleanPivotable: CatalystPivotable[Boolean] = of[Boolean]\n implicit val framelessStringPivotable : CatalystPivotable[String] = of[String]\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystRound.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/** Spark does not return always long on round\n */\n@implicitNotFound(\"Cannot compute round on type ${In}.\")\ntrait CatalystRound[In, Out]\n\nobject CatalystRound {\n private[this] val theInstance = new CatalystRound[Any, Any] {}\n private[this] def of[In, Out]: CatalystRound[In, Out] = theInstance.asInstanceOf[CatalystRound[In, Out]]\n\n implicit val framelessBigDecimal: CatalystRound[BigDecimal, java.math.BigDecimal] = of[BigDecimal, java.math.BigDecimal]\n implicit val framelessDouble : CatalystRound[Double, Long] = of[Double, Long]\n implicit val framelessInt : CatalystRound[Int, Long] = of[Int, Long]\n implicit val framelessLong : CatalystRound[Long, Long] = of[Long, Long]\n implicit val framelessShort : CatalystRound[Short, Long] = of[Short, Long]\n}" }, { "path": "core/src/main/scala/frameless/CatalystSummable.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/**\n * When summing Spark doesn't change these types:\n * - Long -> Long\n * - BigDecimal -> BigDecimal\n * - Double -> Double\n *\n * For other types there are conversions:\n * - Int -> Long\n * - Short -> Long\n */\n@implicitNotFound(\"Cannot compute sum of type ${In}.\")\ntrait CatalystSummable[In, Out] {\n def zero: In\n}\n\nobject CatalystSummable {\n def apply[In, Out](zero: In): CatalystSummable[In, Out] = {\n val _zero = zero\n new CatalystSummable[In, Out] { val zero: In = _zero }\n }\n\n implicit val framelessSummableLong : CatalystSummable[Long, Long] = CatalystSummable(zero = 0L)\n implicit val framelessSummableBigDecimal: CatalystSummable[BigDecimal, BigDecimal] = CatalystSummable(zero = BigDecimal(0))\n implicit val framelessSummableDouble : CatalystSummable[Double, Double] = CatalystSummable(zero = 0.0)\n implicit val framelessSummableInt : CatalystSummable[Int, Long] = CatalystSummable(zero = 0)\n implicit val framelessSummableShort : CatalystSummable[Short, Long] = CatalystSummable(zero = 0)\n}\n" }, { "path": "core/src/main/scala/frameless/CatalystVariance.scala", "content": "package frameless\n\nimport scala.annotation.implicitNotFound\n\n/**\n * Spark's variance and stddev functions always return Double\n */\n@implicitNotFound(\"Cannot compute variance on type ${A}.\")\ntrait CatalystVariance[A]\n\nobject CatalystVariance {\n private[this] val theInstance = new CatalystVariance[Any] {}\n private[this] def of[A]: CatalystVariance[A] = theInstance.asInstanceOf[CatalystVariance[A]]\n\n implicit val framelessIntVariance : CatalystVariance[Int] = of[Int]\n implicit val framelessLongVariance : CatalystVariance[Long] = of[Long]\n implicit val framelessShortVariance : CatalystVariance[Short] = of[Short]\n implicit val framelessBigDecimalVariance: CatalystVariance[BigDecimal] = of[BigDecimal]\n implicit val framelessDoubleVariance : CatalystVariance[Double] = of[Double]\n}\n" }, { "path": "core/src/main/scala/frameless/Injection.scala", "content": "package frameless\n\n/**\n * An Injection[A, B] is a reversible function from A to B.\n *\n * Must obey `forAll { a: A => invert(apply(a)) == a }`.\n */\ntrait Injection[A, B] extends Serializable {\n def apply(a: A): B\n def invert(b: B): A\n}\n\nobject Injection {\n\n def apply[A, B](f: A => B, g: B => A): Injection[A, B] = new Injection[A, B] {\n def apply(a: A): B = f(a)\n def invert(b: B): A = g(b)\n }\n}\n" }, { "path": "core/src/main/scala/frameless/SQLDate.scala", "content": "package frameless\n\n/**\n * Type for the internal Spark representation of SQL date. If the `spark.sql.functions` where typed,\n * [date_add][1] would for instance be defined as `def date_add(d: SQLDate, i: Int); SQLDate`.\n *\n * [1]: https://spark.apache.org/docs/2.0.2/api/java/org/apache/spark/sql/functions.html#add_months(org.apache.spark.sql.Column,%20int)\n */\ncase class SQLDate(days: Int)\n" }, { "path": "core/src/main/scala/frameless/SQLTimestamp.scala", "content": "package frameless\n\n/**\n * Type for the Spark internal representation of a timestamp. If the `spark.sql.functions` where typed,\n * [current_timestamp][1] would for instance be defined as `def current_timestamp(): SQLTimestamp`.\n *\n * [1]: https://spark.apache.org/docs/1.6.2/api/java/org/apache/spark/sql/functions.html#current_timestamp()\n */\ncase class SQLTimestamp(us: Long)\n" }, { "path": "dataset/src/main/scala/frameless/FramelessSyntax.scala", "content": "package frameless\n\nimport org.apache.spark.sql.{Column, DataFrame, Dataset}\n\ntrait FramelessSyntax {\n implicit class ColumnSyntax(self: Column) {\n def typedColumn[T, U: TypedEncoder]: TypedColumn[T, U] = new TypedColumn[T, U](self)\n def typedAggregate[T, U: TypedEncoder]: TypedAggregate[T, U] = new TypedAggregate[T, U](self)\n }\n\n implicit class DatasetSyntax[T: TypedEncoder](self: Dataset[T]) {\n def typed: TypedDataset[T] = TypedDataset.create[T](self)\n }\n\n implicit class DataframeSyntax(self: DataFrame){\n def unsafeTyped[T: TypedEncoder]: TypedDataset[T] = TypedDataset.createUnsafe(self)\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/InjectionEnum.scala", "content": "package frameless\n\nimport shapeless._\n\ntrait InjectionEnum {\n implicit val cnilInjectionEnum: Injection[CNil, String] =\n Injection(\n // $COVERAGE-OFF$No value of type CNil so impossible to test\n _ => throw new Exception(\"Impossible\"),\n // $COVERAGE-ON$\n name =>\n throw new IllegalArgumentException(\n s\"Cannot construct a value of type CNil: $name did not match data constructor names\"\n )\n )\n\n implicit def coproductInjectionEnum[H, T <: Coproduct](\n implicit\n typeable: Typeable[H] ,\n gen: Generic.Aux[H, HNil],\n tInjectionEnum: Injection[T, String]\n ): Injection[H :+: T, String] = {\n val dataConstructorName = typeable.describe.takeWhile(_ != '.')\n\n Injection(\n {\n case Inl(_) => dataConstructorName\n case Inr(t) => tInjectionEnum.apply(t)\n },\n { name =>\n if (name == dataConstructorName)\n Inl(gen.from(HNil))\n else\n Inr(tInjectionEnum.invert(name))\n }\n )\n }\n\n implicit def genericInjectionEnum[A, R](\n implicit\n gen: Generic.Aux[A, R],\n rInjectionEnum: Injection[R, String]\n ): Injection[A, String] =\n Injection(\n value => rInjectionEnum(gen.to(value)),\n name => gen.from(rInjectionEnum.invert(name))\n )\n}\n" }, { "path": "dataset/src/main/scala/frameless/IsValueClass.scala", "content": "package frameless\n\nimport shapeless._\nimport shapeless.labelled.FieldType\n\n/** Evidence that `T` is a Value class */\n@annotation.implicitNotFound(msg = \"${T} is not a Value class\")\nfinal class IsValueClass[T] private() {}\n\nobject IsValueClass {\n /** Provides an evidence `A` is a Value class */\n implicit def apply[A <: AnyVal, G <: ::[_, HNil], H <: ::[_ <: FieldType[_ <: Symbol, _], HNil]](\n implicit\n i0: LabelledGeneric.Aux[A, G],\n i1: DropUnitValues.Aux[G, H]): IsValueClass[A] = new IsValueClass[A]\n\n}\n" }, { "path": "dataset/src/main/scala/frameless/Job.scala", "content": "package frameless\n\nimport org.apache.spark.sql.SparkSession\n\nsealed abstract class Job[A](implicit spark: SparkSession) { self =>\n /** Runs a new Spark job. */\n def run(): A\n\n def withGroupId(groupId: String): Job[A] = {\n withLocalProperty(\"spark.jobGroup.id\", groupId)\n }\n\n def withDescription(groupId: String): Job[A] = {\n withLocalProperty(\"spark.job.description\", groupId)\n }\n\n def withLocalProperty(key: String, value: String): Job[A] = {\n new Job[A] {\n def run(): A = {\n spark.sparkContext.setLocalProperty(key, value)\n self.run()\n }\n }\n }\n\n def map[B](fn: A => B): Job[B] = new Job[B]()(spark) {\n def run(): B = fn(Job.this.run())\n }\n\n def flatMap[B](fn: A => Job[B]): Job[B] = new Job[B]()(spark) {\n def run(): B = fn(Job.this.run()).run()\n }\n}\n\n\nobject Job {\n def apply[A](a: => A)(implicit spark: SparkSession): Job[A] = new Job[A] {\n def run(): A = a\n }\n\n implicit val framelessSparkDelayForJob: SparkDelay[Job] = new SparkDelay[Job] {\n def delay[A](a: => A)(implicit spark: SparkSession): Job[A] = Job(a)\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/RecordEncoder.scala", "content": "package frameless\n\nimport org.apache.spark.sql.FramelessInternals\n\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.catalyst.expressions.objects.{\n Invoke, NewInstance, UnwrapOption, WrapOption\n}\nimport org.apache.spark.sql.types._\n\nimport shapeless._\nimport shapeless.labelled.FieldType\nimport shapeless.ops.hlist.IsHCons\nimport shapeless.ops.record.Keys\n\nimport scala.reflect.ClassTag\n\ncase class RecordEncoderField(\n ordinal: Int,\n name: String,\n encoder: TypedEncoder[_]\n)\n\ntrait RecordEncoderFields[T <: HList] extends Serializable {\n def value: List[RecordEncoderField]\n\n override def toString: String =\n s\"\"\"RecordEncoderFields${value.mkString(\"[\", \", \", \"]\")}\"\"\"\n}\n\nobject RecordEncoderFields {\n\n implicit def deriveRecordLast[K <: Symbol, H]\n (implicit\n key: Witness.Aux[K],\n head: RecordFieldEncoder[H]\n ): RecordEncoderFields[FieldType[K, H] :: HNil] = new RecordEncoderFields[FieldType[K, H] :: HNil] {\n def value: List[RecordEncoderField] = fieldEncoder[K, H] :: Nil\n }\n\n implicit def deriveRecordCons[K <: Symbol, H, T <: HList]\n (implicit\n key: Witness.Aux[K],\n head: RecordFieldEncoder[H],\n tail: RecordEncoderFields[T]\n ): RecordEncoderFields[FieldType[K, H] :: T] = new RecordEncoderFields[FieldType[K, H] :: T] {\n def value: List[RecordEncoderField] =\n fieldEncoder[K, H] :: tail.value.map(x => x.copy(ordinal = x.ordinal + 1))\n }\n\n private def fieldEncoder[K <: Symbol, H](implicit key: Witness.Aux[K], e: RecordFieldEncoder[H]): RecordEncoderField = RecordEncoderField(0, key.value.name, e.encoder)\n}\n\n/**\n * Assists the generation of constructor call parameters from a labelled generic representation.\n * As Unit typed fields were removed earlier, we need to put back unit literals in the appropriate positions.\n *\n * @tparam T labelled generic representation of type fields\n */\ntrait NewInstanceExprs[T <: HList] extends Serializable {\n def from(exprs: List[Expression]): Seq[Expression]\n}\n\nobject NewInstanceExprs {\n\n implicit def deriveHNil: NewInstanceExprs[HNil] = new NewInstanceExprs[HNil] {\n def from(exprs: List[Expression]): Seq[Expression] = Nil\n }\n\n implicit def deriveUnit[K <: Symbol, T <: HList]\n (implicit\n tail: NewInstanceExprs[T]\n ): NewInstanceExprs[FieldType[K, Unit] :: T] = new NewInstanceExprs[FieldType[K, Unit] :: T] {\n def from(exprs: List[Expression]): Seq[Expression] =\n Literal.fromObject(()) +: tail.from(exprs)\n }\n\n implicit def deriveNonUnit[K <: Symbol, V, T <: HList]\n (implicit\n notUnit: V =:!= Unit,\n tail: NewInstanceExprs[T]\n ): NewInstanceExprs[FieldType[K, V] :: T] = new NewInstanceExprs[FieldType[K, V] :: T] {\n def from(exprs: List[Expression]): Seq[Expression] = exprs.head +: tail.from(exprs.tail)\n }\n}\n\n/**\n * Drops fields with Unit type from labelled generic representation of types.\n *\n * @tparam L labelled generic representation of type fields\n */\ntrait DropUnitValues[L <: HList] extends DepFn1[L] with Serializable { type Out <: HList }\n\nobject DropUnitValues {\n def apply[L <: HList](implicit dropUnitValues: DropUnitValues[L]): Aux[L, dropUnitValues.Out] = dropUnitValues\n\n type Aux[L <: HList, Out0 <: HList] = DropUnitValues[L] { type Out = Out0 }\n\n implicit def deriveHNil[H]: Aux[HNil, HNil] = new DropUnitValues[HNil] {\n type Out = HNil\n def apply(l: HNil): Out = HNil\n }\n\n implicit def deriveUnit[K <: Symbol, T <: HList, OutT <: HList]\n (implicit\n dropUnitValues : DropUnitValues.Aux[T, OutT]\n ): Aux[FieldType[K, Unit] :: T, OutT] = new DropUnitValues[FieldType[K, Unit] :: T] {\n type Out = OutT\n def apply(l : FieldType[K, Unit] :: T): Out = dropUnitValues(l.tail)\n }\n\n implicit def deriveNonUnit[K <: Symbol, V, T <: HList, OutH, OutT <: HList]\n (implicit\n nonUnit: V =:!= Unit,\n dropUnitValues : DropUnitValues.Aux[T, OutT]\n ): Aux[FieldType[K, V] :: T, FieldType[K, V] :: OutT] = new DropUnitValues[FieldType[K, V] :: T] {\n type Out = FieldType[K, V] :: OutT\n def apply(l : FieldType[K, V] :: T): Out = l.head :: dropUnitValues(l.tail)\n }\n}\n\nclass RecordEncoder[F, G <: HList, H <: HList]\n (implicit\n i0: LabelledGeneric.Aux[F, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons[H],\n fields: Lazy[RecordEncoderFields[H]],\n newInstanceExprs: Lazy[NewInstanceExprs[G]],\n classTag: ClassTag[F]\n ) extends TypedEncoder[F] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = FramelessInternals.objectTypeFor[F]\n\n def catalystRepr: DataType = {\n val structFields = fields.value.value.map { field =>\n StructField(\n name = field.name,\n dataType = field.encoder.catalystRepr,\n nullable = field.encoder.nullable,\n metadata = Metadata.empty\n )\n }\n\n StructType(structFields)\n }\n\n def toCatalyst(path: Expression): Expression = {\n val nameExprs = fields.value.value.map { field =>\n Literal(field.name)\n }\n\n val valueExprs = fields.value.value.map { field =>\n val fieldPath = Invoke(path, field.name, field.encoder.jvmRepr, Nil)\n field.encoder.toCatalyst(fieldPath)\n }\n\n // the way exprs are encoded in CreateNamedStruct\n val exprs = nameExprs.zip(valueExprs).flatMap {\n case (nameExpr, valueExpr) => nameExpr :: valueExpr :: Nil\n }\n\n val createExpr = CreateNamedStruct(exprs)\n val nullExpr = Literal.create(null, createExpr.dataType)\n\n If(IsNull(path), nullExpr, createExpr)\n }\n\n def fromCatalyst(path: Expression): Expression = {\n val exprs = fields.value.value.map { field =>\n field.encoder.fromCatalyst(\n GetStructField(path, field.ordinal, Some(field.name)))\n }\n\n val newArgs = newInstanceExprs.value.from(exprs)\n val newExpr = NewInstance(\n classTag.runtimeClass, newArgs, jvmRepr, propagateNull = true)\n\n val nullExpr = Literal.create(null, jvmRepr)\n\n If(IsNull(path), nullExpr, newExpr)\n }\n}\n\nfinal class RecordFieldEncoder[T](\n val encoder: TypedEncoder[T],\n private[frameless] val jvmRepr: DataType,\n private[frameless] val fromCatalyst: Expression => Expression,\n private[frameless] val toCatalyst: Expression => Expression\n) extends Serializable\n\nobject RecordFieldEncoder extends RecordFieldEncoderLowPriority {\n\n /**\n * @tparam F the value class\n * @tparam G the single field of the value class\n * @tparam H the single field of the value class (with guarantee it's not a `Unit` value)\n * @tparam K the key type for the fields\n * @tparam V the inner value type\n */\n implicit def optionValueClass[F : IsValueClass, G <: ::[_, HNil], H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], K <: Symbol, V, KS <: ::[_ <: Symbol, HNil]]\n (implicit\n i0: LabelledGeneric.Aux[F, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil],\n i3: Keys.Aux[H, KS],\n i4: IsHCons.Aux[KS, K, HNil],\n i5: TypedEncoder[V],\n i6: ClassTag[F]\n ): RecordFieldEncoder[Option[F]] = {\n val fieldName = i4.head(i3()).name\n val innerJvmRepr = ObjectType(i6.runtimeClass)\n\n val catalyst: Expression => Expression = { path =>\n val value = UnwrapOption(innerJvmRepr, path)\n val javaValue = Invoke(value, fieldName, i5.jvmRepr, Nil)\n\n i5.toCatalyst(javaValue)\n }\n\n val fromCatalyst: Expression => Expression = { path =>\n val javaValue = i5.fromCatalyst(path)\n val value = NewInstance(i6.runtimeClass, Seq(javaValue), innerJvmRepr)\n\n WrapOption(value, innerJvmRepr)\n }\n\n val jvmr = ObjectType(classOf[Option[F]])\n\n new RecordFieldEncoder[Option[F]](\n encoder = new TypedEncoder[Option[F]] {\n val nullable = true\n\n val jvmRepr = jvmr\n\n @inline def catalystRepr: DataType = i5.catalystRepr\n\n def fromCatalyst(path: Expression): Expression = {\n val javaValue = i5.fromCatalyst(path)\n val value = NewInstance(\n i6.runtimeClass, Seq(javaValue), innerJvmRepr)\n\n WrapOption(value, innerJvmRepr)\n }\n\n def toCatalyst(path: Expression): Expression = catalyst(path)\n\n override def toString: String = s\"RecordFieldEncoder.optionValueClass[${i6.runtimeClass.getName}]('${fieldName}', $i5)\"\n },\n jvmRepr = jvmr,\n fromCatalyst = fromCatalyst,\n toCatalyst = catalyst\n )\n }\n\n /**\n * @tparam F the value class\n * @tparam G the single field of the value class\n * @tparam H the single field of the value class (with guarantee it's not a `Unit` value)\n * @tparam V the inner value type\n */\n implicit def valueClass[F : IsValueClass, G <: ::[_, HNil], H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], K <: Symbol, V, KS <: ::[_ <: Symbol, HNil]]\n (implicit\n i0: LabelledGeneric.Aux[F, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil],\n i3: Keys.Aux[H, KS],\n i4: IsHCons.Aux[KS, K, HNil],\n i5: TypedEncoder[V],\n i6: ClassTag[F]\n ): RecordFieldEncoder[F] = {\n val cls = i6.runtimeClass\n val jvmr = i5.jvmRepr\n val fieldName = i4.head(i3()).name\n\n new RecordFieldEncoder[F](\n encoder = new TypedEncoder[F] {\n def nullable = i5.nullable\n\n def jvmRepr = jvmr\n\n def catalystRepr: DataType = i5.catalystRepr\n\n def fromCatalyst(path: Expression): Expression =\n i5.fromCatalyst(path)\n\n @inline def toCatalyst(path: Expression): Expression =\n i5.toCatalyst(path)\n\n override def toString: String = s\"RecordFieldEncoder.valueClass[${cls.getName}]('${fieldName}', ${i5})\"\n },\n jvmRepr = FramelessInternals.objectTypeFor[F],\n fromCatalyst = { expr: Expression =>\n NewInstance(\n i6.runtimeClass,\n i5.fromCatalyst(expr) :: Nil,\n ObjectType(i6.runtimeClass))\n },\n toCatalyst = { expr: Expression =>\n i5.toCatalyst(Invoke(expr, fieldName, jvmr))\n }\n )\n }\n}\n\nprivate[frameless] sealed trait RecordFieldEncoderLowPriority {\n implicit def apply[T](implicit e: TypedEncoder[T]): RecordFieldEncoder[T] = new RecordFieldEncoder[T](e, e.jvmRepr, e.fromCatalyst, e.toCatalyst)\n}\n" }, { "path": "dataset/src/main/scala/frameless/SparkDelay.scala", "content": "package frameless\n\nimport org.apache.spark.sql.SparkSession\n\ntrait SparkDelay[F[_]] {\n def delay[A](a: => A)(implicit spark: SparkSession): F[A]\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedColumn.scala", "content": "package frameless\n\nimport frameless.functions.{litAggr, lit => flit}\nimport frameless.syntax._\n\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.types.DecimalType\nimport org.apache.spark.sql.{Column, FramelessInternals}\n\nimport shapeless._\nimport shapeless.ops.record.Selector\n\nimport scala.annotation.implicitNotFound\nimport scala.reflect.ClassTag\n\nimport scala.language.experimental.macros\n\nsealed trait UntypedExpression[T] {\n def expr: Expression\n def uencoder: TypedEncoder[_]\n override def toString: String = expr.toString()\n}\n\n/** Expression used in `select`-like constructions.\n */\nsealed class TypedColumn[T, U](expr: Expression)(\n implicit val uenc: TypedEncoder[U]\n) extends AbstractTypedColumn[T, U](expr) {\n\n type ThisType[A, B] = TypedColumn[A, B]\n\n def this(column: Column)(implicit uencoder: TypedEncoder[U]) =\n this(FramelessInternals.expr(column))\n\n override def typed[W, U1: TypedEncoder](c: Column): TypedColumn[W, U1] = c.typedColumn\n\n override def lit[U1: TypedEncoder](c: U1): TypedColumn[T, U1] = flit(c)\n}\n\n/** Expression used in `agg`-like constructions.\n */\nsealed class TypedAggregate[T, U](expr: Expression)(\n implicit val uenc: TypedEncoder[U]\n) extends AbstractTypedColumn[T, U](expr) {\n\n type ThisType[A, B] = TypedAggregate[A, B]\n\n def this(column: Column)(implicit uencoder: TypedEncoder[U]) = {\n this(FramelessInternals.expr(column))\n }\n\n override def typed[W, U1: TypedEncoder](c: Column): TypedAggregate[W, U1] = c.typedAggregate\n\n override def lit[U1: TypedEncoder](c: U1): TypedAggregate[T, U1] = litAggr(c)\n}\n\n/** Generic representation of a typed column. A typed column can either be a [[TypedAggregate]] or\n * a [[frameless.TypedColumn]].\n *\n * Documentation marked \"apache/spark\" is thanks to apache/spark Contributors\n * at https://github.com/apache/spark, licensed under Apache v2.0 available at\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * @tparam T phantom type representing the dataset on which this columns is\n * selected. When `T = A with B` the selection is on either A or B.\n * @tparam U type of column\n */\nabstract class AbstractTypedColumn[T, U]\n (val expr: Expression)\n (implicit val uencoder: TypedEncoder[U])\n extends UntypedExpression[T] { self =>\n\n type ThisType[A, B] <: AbstractTypedColumn[A, B]\n\n /** A helper class to make to simplify working with Optional fields.\n *\n * {{{\n * val x: TypedColumn[Option[Int]] = _\n * x.opt.map(_*2) // This only compiles if the type of x is Option[X] (in this example X is of type Int)\n * }}}\n *\n * @note Known issue: map() will NOT work when the applied function is a udf().\n * It will compile and then throw a runtime error.\n **/\n trait Mapper[X] {\n def map[G, OutputType[_,_]](u: ThisType[T, X] => OutputType[T,G])\n (implicit\n ev: OutputType[T,G] <:< AbstractTypedColumn[T, G]\n ): OutputType[T, Option[G]] = {\n u(self.asInstanceOf[ThisType[T, X]]).asInstanceOf[OutputType[T, Option[G]]]\n }\n }\n\n /** Makes it easier to work with Optional columns. It returns an instance of `Mapper[X]`\n * where `X` is type of the unwrapped Optional. E.g., in the case of `Option[Long]`,\n * `X` is of type Long.\n *\n * {{{\n * val x: TypedColumn[Option[Int]] = _\n * x.opt.map(_*2)\n * }}}\n * */\n def opt[X](implicit x: U <:< Option[X]): Mapper[X] = new Mapper[X] {}\n\n /** Fall back to an untyped Column */\n def untyped: Column = new Column(expr)\n\n private def equalsTo[TT, W](other: ThisType[TT, U])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] = typed {\n if (uencoder.nullable) EqualNullSafe(self.expr, other.expr)\n else EqualTo(self.expr, other.expr)\n }\n\n /** Creates a typed column of either TypedColumn or TypedAggregate from an expression. */\n protected def typed[W, U1: TypedEncoder](e: Expression): ThisType[W, U1] =\n typed(new Column(e))\n\n /** Creates a typed column of either TypedColumn or TypedAggregate. */\n def typed[W, U1: TypedEncoder](c: Column): ThisType[W, U1]\n\n /** Creates a typed column of either TypedColumn or TypedAggregate. */\n def lit[U1: TypedEncoder](c: U1): ThisType[T, U1]\n\n /** Equality test.\n * {{{\n * df.filter( df.col('a) === 1 )\n * }}}\n *\n * apache/spark\n */\n def ===(u: U): ThisType[T, Boolean] =\n equalsTo(lit(u))\n\n /** Equality test.\n * {{{\n * df.filter( df.col('a) === df.col('b) )\n * }}}\n *\n * apache/spark\n */\n def ===[TT, W](other: ThisType[TT, U])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n equalsTo(other)\n\n /** Inequality test.\n * \n * {{{\n * df.filter(df.col('a) =!= df.col('b))\n * }}}\n *\n * apache/spark\n */\n def =!=[TT, W](other: ThisType[TT, U])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(Not(equalsTo(other).expr))\n\n /** Inequality test.\n * \n * {{{\n * df.filter(df.col('a) =!= \"a\")\n * }}}\n *\n * apache/spark\n */\n def =!=(u: U): ThisType[T, Boolean] = typed(Not(equalsTo(lit(u)).expr))\n\n /** True if the current expression is an Option and it's None.\n *\n * apache/spark\n */\n def isNone(implicit i0: U <:< Option[_]): ThisType[T, Boolean] =\n typed(IsNull(expr))\n\n /** True if the current expression is an Option and it's not None.\n *\n * apache/spark\n */\n def isNotNone(implicit i0: U <:< Option[_]): ThisType[T, Boolean] =\n typed(IsNotNull(expr))\n\n /** True if the current expression is a fractional number and is not NaN.\n *\n * apache/spark\n */\n def isNaN(implicit n: CatalystNaN[U]): ThisType[T, Boolean] =\n typed(self.untyped.isNaN)\n\n /**\n * True if the value for this optional column `exists` as expected\n * (see `Option.exists`).\n * \n * {{{\n * df.col('opt).isSome(_ === someOtherCol)\n * }}}\n */\n def isSome[V](exists: ThisType[T, V] => ThisType[T, Boolean])(implicit i0: U <:< Option[V]): ThisType[T, Boolean] = someOr[V](exists, false)\n\n /**\n * True if the value for this optional column `exists` as expected,\n * or is `None`. (see `Option.forall`).\n * \n * {{{\n * df.col('opt).isSomeOrNone(_ === someOtherCol)\n * }}}\n */\n def isSomeOrNone[V](exists: ThisType[T, V] => ThisType[T, Boolean])(implicit i0: U <:< Option[V]): ThisType[T, Boolean] = someOr[V](exists, true)\n\n private def someOr[V](exists: ThisType[T, V] => ThisType[T, Boolean], default: Boolean)(implicit i0: U <:< Option[V]): ThisType[T, Boolean] = {\n val defaultExpr = if (default) Literal.TrueLiteral else Literal.FalseLiteral\n\n typed(Coalesce(Seq(opt(i0).map(exists).expr, defaultExpr)))\n }\n\n /** Convert an Optional column by providing a default value.\n * \n * {{{\n * df(df('opt).getOrElse(df('defaultValue)))\n * }}}\n */\n def getOrElse[TT, W, Out](default: ThisType[TT, Out])(implicit i0: U =:= Option[Out], i1: With.Aux[T, TT, W]): ThisType[W, Out] =\n typed(Coalesce(Seq(expr, default.expr)))(default.uencoder)\n\n /** Convert an Optional column by providing a default value.\n * \n * {{{\n * df( df('opt).getOrElse(defaultConstant) )\n * }}}\n */\n def getOrElse[Out: TypedEncoder](default: Out)(implicit i0: U =:= Option[Out]): ThisType[T, Out] =\n getOrElse(lit[Out](default))\n\n /** Sum of this expression and another expression.\n * \n * {{{\n * // The following selects the sum of a person's height and weight.\n * people.select( people.col('height) plus people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def plus[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n typed(self.untyped.plus(other.untyped))\n\n /** Sum of this expression and another expression.\n * {{{\n * // The following selects the sum of a person's height and weight.\n * people.select( people.col('height) + people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def +[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n plus(other)\n\n /** Sum of this expression (column) with a constant.\n * {{{\n * // The following selects the sum of a person's height and weight.\n * people.select( people('height) + 2 )\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def +(u: U)(implicit n: CatalystNumeric[U]): ThisType[T, U] =\n typed(self.untyped.plus(u))\n\n /**\n * Inversion of boolean expression, i.e. NOT.\n * {{{\n * // Select rows that are not active (isActive === false)\n * df.filter( !df('isActive) )\n * }}}\n *\n * apache/spark\n */\n def unary_!(implicit i0: U <:< Boolean): ThisType[T, Boolean] =\n typed(!untyped)\n\n /** Unary minus, i.e. negate the expression.\n * {{{\n * // Select the amount column and negates all values.\n * df.select( -df('amount) )\n * }}}\n *\n * apache/spark\n */\n def unary_-(implicit n: CatalystNumeric[U]): ThisType[T, U] =\n typed(-self.untyped)\n\n /** Subtraction. Subtract the other expression from this expression.\n * {{{\n * // The following selects the difference between people's height and their weight.\n * people.select( people.col('height) minus people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def minus[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n typed(self.untyped.minus(other.untyped))\n\n /** Subtraction. Subtract the other expression from this expression.\n * {{{\n * // The following selects the difference between people's height and their weight.\n * people.select( people.col('height) - people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def -[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n minus(other)\n\n /** Subtraction. Subtract the other expression from this expression.\n * {{{\n * // The following selects the difference between people's height and their weight.\n * people.select( people('height) - 1 )\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def -(u: U)(implicit n: CatalystNumeric[U]): ThisType[T, U] =\n typed(self.untyped.minus(u))\n\n /** Multiplication of this expression and another expression.\n * {{{\n * // The following multiplies a person's height by their weight.\n * people.select( people.col('height) multiply people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def multiply[TT, W]\n (other: ThisType[TT, U])\n (implicit\n n: CatalystNumeric[U],\n w: With.Aux[T, TT, W],\n t: ClassTag[U]\n ): ThisType[W, U] = typed {\n if (t.runtimeClass == BigDecimal(0).getClass) {\n // That's apparently the only way to get sound multiplication.\n // See https://issues.apache.org/jira/browse/SPARK-22036\n val dt = DecimalType(20, 14)\n self.untyped.cast(dt).multiply(other.untyped.cast(dt))\n } else {\n self.untyped.multiply(other.untyped)\n }\n }\n\n /** Multiplication of this expression and another expression.\n * {{{\n * // The following multiplies a person's height by their weight.\n * people.select( people.col('height) * people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def *[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W], t: ClassTag[U]): ThisType[W, U] =\n multiply(other)\n\n /** Multiplication of this expression a constant.\n * {{{\n * // The following multiplies a person's height by their weight.\n * people.select( people.col('height) * people.col('weight) )\n * }}}\n *\n * apache/spark\n */\n def *(u: U)(implicit n: CatalystNumeric[U]): ThisType[T, U] =\n typed(self.untyped.multiply(u))\n\n /** Modulo (a.k.a. remainder) expression.\n *\n * apache/spark\n */\n def mod[Out: TypedEncoder, TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, Out] =\n typed(self.untyped.mod(other.untyped))\n\n /** Modulo (a.k.a. remainder) expression.\n *\n * apache/spark\n */\n def %[TT, W](other: ThisType[TT, U])(implicit n: CatalystNumeric[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n mod(other)\n\n /** Modulo (a.k.a. remainder) expression.\n *\n * apache/spark\n */\n def %(u: U)(implicit n: CatalystNumeric[U]): ThisType[T, U] =\n typed(self.untyped.mod(u))\n\n /** Division this expression by another expression.\n * {{{\n * // The following divides a person's height by their weight.\n * people.select( people('height) / people('weight) )\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def divide[Out: TypedEncoder, TT, W](other: ThisType[TT, U])(implicit n: CatalystDivisible[U, Out], w: With.Aux[T, TT, W]): ThisType[W, Out] =\n typed(self.untyped.divide(other.untyped))\n\n /** Division this expression by another expression.\n * {{{\n * // The following divides a person's height by their weight.\n * people.select( people('height) / people('weight) )\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def /[Out, TT, W](other: ThisType[TT, U])(implicit n: CatalystDivisible[U, Out], e: TypedEncoder[Out], w: With.Aux[T, TT, W]): ThisType[W, Out] =\n divide(other)\n\n /** Division this expression by another expression.\n * {{{\n * // The following divides a person's height by their weight.\n * people.select( people('height) / 2 )\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def /(u: U)(implicit n: CatalystNumeric[U]): ThisType[T, Double] =\n typed(self.untyped.divide(u))\n\n /** Returns a descending ordering used in sorting\n *\n * apache/spark\n */\n def desc(implicit catalystOrdered: CatalystOrdered[U]): SortedTypedColumn[T, U] =\n new SortedTypedColumn[T, U](untyped.desc)\n\n /** Returns an ascending ordering used in sorting\n *\n * apache/spark\n */\n def asc(implicit catalystOrdered: CatalystOrdered[U]): SortedTypedColumn[T, U] =\n new SortedTypedColumn[T, U](untyped.asc)\n\n /** Bitwise AND this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseAND (df.col('colB)))\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def bitwiseAND(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n typed(self.untyped.bitwiseAND(u))\n\n /** Bitwise AND this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseAND (df.col('colB)))\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def bitwiseAND[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n typed(self.untyped.bitwiseAND(other.untyped))\n\n /** Bitwise AND this expression and another expression (of same type).\n * {{{\n * df.select(df.col('colA).cast[Int] & -1)\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def &(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n bitwiseAND(u)\n\n /** Bitwise AND this expression and another expression.\n * {{{\n * df.select(df.col('colA) & (df.col('colB)))\n * }}}\n *\n * @param other a constant of the same type\n * apache/spark\n */\n def &[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n bitwiseAND(other)\n\n /** Bitwise OR this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseOR (df.col('colB)))\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def bitwiseOR(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n typed(self.untyped.bitwiseOR(u))\n\n /** Bitwise OR this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseOR (df.col('colB)))\n * }}}\n *\n * @param other a constant of the same type\n * apache/spark\n */\n def bitwiseOR[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n typed(self.untyped.bitwiseOR(other.untyped))\n\n /** Bitwise OR this expression and another expression (of same type).\n * {{{\n * df.select(df.col('colA).cast[Long] | 1L)\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def |(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n bitwiseOR(u)\n\n /** Bitwise OR this expression and another expression.\n * {{{\n * df.select(df.col('colA) | (df.col('colB)))\n * }}}\n *\n * @param other a constant of the same type\n * apache/spark\n */\n def |[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n bitwiseOR(other)\n\n /** Bitwise XOR this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseXOR (df.col('colB)))\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def bitwiseXOR(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n typed(self.untyped.bitwiseXOR(u))\n\n /** Bitwise XOR this expression and another expression.\n * {{{\n * df.select(df.col('colA) bitwiseXOR (df.col('colB)))\n * }}}\n *\n * @param other a constant of the same type\n * apache/spark\n */\n def bitwiseXOR[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n typed(self.untyped.bitwiseXOR(other.untyped))\n\n /** Bitwise XOR this expression and another expression (of same type).\n * {{{\n * df.select(df.col('colA).cast[Long] ^ 1L)\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def ^(u: U)(implicit n: CatalystBitwise[U]): ThisType[T, U] =\n bitwiseXOR(u)\n\n /** Bitwise XOR this expression and another expression.\n * {{{\n * df.select(df.col('colA) ^ (df.col('colB)))\n * }}}\n *\n * @param other a constant of the same type\n * apache/spark\n */\n def ^[TT, W](other: ThisType[TT, U])(implicit n: CatalystBitwise[U], w: With.Aux[T, TT, W]): ThisType[W, U] =\n bitwiseXOR(other)\n\n /** Casts the column to a different type.\n * {{{\n * df.select(df('a).cast[Int])\n * }}}\n */\n def cast[A: TypedEncoder](implicit c: CatalystCast[U, A]): ThisType[T, A] =\n typed(self.untyped.cast(TypedEncoder[A].catalystRepr))\n\n /**\n * An expression that returns a substring\n * {{{\n * df.select(df('a).substr(0, 5))\n * }}}\n *\n * @param startPos starting position\n * @param len length of the substring\n */\n def substr(startPos: Int, len: Int)(implicit ev: U =:= String): ThisType[T, String] =\n typed(self.untyped.substr(startPos, len))\n\n /**\n * An expression that returns a substring\n * {{{\n * df.select(df('a).substr(df('b), df('c)))\n * }}}\n *\n * @param startPos expression for the starting position\n * @param len expression for the length of the substring\n */\n def substr[TT1, TT2, W1, W2](startPos: ThisType[TT1, Int], len: ThisType[TT2, Int])\n (implicit\n ev: U =:= String,\n w1: With.Aux[T, TT1, W1],\n w2: With.Aux[W1, TT2, W2]): ThisType[W2, String] =\n typed(self.untyped.substr(startPos.untyped, len.untyped))\n\n /** SQL like expression. Returns a boolean column based on a SQL LIKE match.\n * {{{\n * val ds = TypedDataset.create(X2(\"foo\", \"bar\") :: Nil)\n * // true\n * ds.select(ds('a).like(\"foo\"))\n *\n * // Selected column has value \"bar\"\n * ds.select(when(ds('a).like(\"f\"), ds('a)).otherwise(ds('b))\n * }}}\n * apache/spark\n */\n def like(literal: String)(implicit ev: U =:= String): ThisType[T, Boolean] =\n typed(self.untyped.like(literal))\n\n /** SQL RLIKE expression (LIKE with Regex). Returns a boolean column based on a regex match.\n * {{{\n * val ds = TypedDataset.create(X1(\"foo\") :: Nil)\n * // true\n * ds.select(ds('a).rlike(\"foo\"))\n *\n * // true\n * ds.select(ds('a).rlike(\".*))\n * }}}\n * apache/spark\n */\n def rlike(literal: String)(implicit ev: U =:= String): ThisType[T, Boolean] =\n typed(self.untyped.rlike(literal))\n\n /** String contains another string literal.\n * {{{\n * df.filter ( df.col('a).contains(\"foo\") )\n * }}}\n *\n * @param other a string that is being tested against.\n * apache/spark\n */\n def contains(other: String)(implicit ev: U =:= String): ThisType[T, Boolean] =\n typed(self.untyped.contains(other))\n\n /** String contains.\n * {{{\n * df.filter ( df.col('a).contains(df.col('b) )\n * }}}\n *\n * @param other a column which values is used as a string that is being tested against.\n * apache/spark\n */\n def contains[TT, W](other: ThisType[TT, U])(implicit ev: U =:= String, w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped.contains(other.untyped))\n\n /** String starts with another string literal.\n * {{{\n * df.filter ( df.col('a).startsWith(\"foo\")\n * }}}\n *\n * @param other a prefix that is being tested against.\n * apache/spark\n */\n def startsWith(other: String)(implicit ev: U =:= String): ThisType[T, Boolean] =\n typed(self.untyped.startsWith(other))\n\n /** String starts with.\n * {{{\n * df.filter ( df.col('a).startsWith(df.col('b))\n * }}}\n *\n * @param other a column which values is used as a prefix that is being tested against.\n * apache/spark\n */\n def startsWith[TT, W](other: ThisType[TT, U])(implicit ev: U =:= String, w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped.startsWith(other.untyped))\n\n /** String ends with another string literal.\n * {{{\n * df.filter ( df.col('a).endsWith(\"foo\")\n * }}}\n *\n * @param other a suffix that is being tested against.\n * apache/spark\n */\n def endsWith(other: String)(implicit ev: U =:= String): ThisType[T, Boolean] =\n typed(self.untyped.endsWith(other))\n\n /** String ends with.\n * {{{\n * df.filter ( df.col('a).endsWith(df.col('b))\n * }}}\n *\n * @param other a column which values is used as a suffix that is being tested against.\n * apache/spark\n */\n def endsWith[TT, W](other: ThisType[TT, U])(implicit ev: U =:= String, w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped.endsWith(other.untyped))\n\n /** Boolean AND.\n * {{{\n * df.filter ( (df.col('a) === 1).and(df.col('b) > 5) )\n * }}}\n */\n def and[TT, W](other: ThisType[TT, Boolean])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped.and(other.untyped))\n\n /** Boolean AND.\n * {{{\n * df.filter ( df.col('a) === 1 && df.col('b) > 5)\n * }}}\n */\n def && [TT, W](other: ThisType[TT, Boolean])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n and(other)\n\n /** Boolean OR.\n * {{{\n * df.filter ( (df.col('a) === 1).or(df.col('b) > 5) )\n * }}}\n */\n def or[TT, W](other: ThisType[TT, Boolean])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped.or(other.untyped))\n\n /** Boolean OR.\n * {{{\n * df.filter ( df.col('a) === 1 || df.col('b) > 5)\n * }}}\n */\n def || [TT, W](other: ThisType[TT, Boolean])(implicit w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n or(other)\n\n /** Less than.\n * \n * {{{\n * // The following selects people younger than the maxAge column.\n * df.select(df('age) < df('maxAge) )\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def <[TT, W](other: ThisType[TT, U])(implicit i0: CatalystOrdered[U], w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped < other.untyped)\n\n /** Less than or equal to.\n * \n * {{{\n * // The following selects people younger or equal than the maxAge column.\n * df.select(df('age) <= df('maxAge)\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def <=[TT, W](other: ThisType[TT, U])(implicit i0: CatalystOrdered[U], w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped <= other.untyped)\n\n /** Greater than.\n * {{{\n * // The following selects people older than the maxAge column.\n * df.select( df('age) > df('maxAge) )\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def >[TT, W](other: ThisType[TT, U])(implicit i0: CatalystOrdered[U], w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped > other.untyped)\n\n /** Greater than or equal.\n * {{{\n * // The following selects people older or equal than the maxAge column.\n * df.select( df('age) >= df('maxAge) )\n * }}}\n *\n * @param other another column of the same type\n * apache/spark\n */\n def >=[TT, W](other: ThisType[TT, U])(implicit i0: CatalystOrdered[U], w: With.Aux[T, TT, W]): ThisType[W, Boolean] =\n typed(self.untyped >= other.untyped)\n\n /** Less than.\n * {{{\n * // The following selects people younger than 21.\n * df.select( df('age) < 21 )\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def <(u: U)(implicit i0: CatalystOrdered[U]): ThisType[T, Boolean] =\n typed(self.untyped < lit(u)(self.uencoder).untyped)\n\n /** Less than or equal to.\n * {{{\n * // The following selects people younger than 22.\n * df.select( df('age) <= 2 )\n * }}}\n *\n * @param u a constant of the same type\n * apache/spark\n */\n def <=(u: U)(implicit i0: CatalystOrdered[U]): ThisType[T, Boolean] =\n typed(self.untyped <= lit(u)(self.uencoder).untyped)\n\n /** Greater than.\n * {{{\n * // The following selects people older than 21.\n * df.select( df('age) > 21 )\n * }}}\n *\n * @param u another column of the same type\n * apache/spark\n */\n def >(u: U)(implicit i0: CatalystOrdered[U]): ThisType[T, Boolean] =\n typed(self.untyped > lit(u)(self.uencoder).untyped)\n\n /** Greater than or equal.\n * {{{\n * // The following selects people older than 20.\n * df.select( df('age) >= 21 )\n * }}}\n *\n * @param u another column of the same type\n * apache/spark\n */\n def >=(u: U)(implicit i0: CatalystOrdered[U]): ThisType[T, Boolean] =\n typed(self.untyped >= lit(u)(self.uencoder).untyped)\n\n /**\n * Returns true if the value of this column is contained in of the arguments.\n * {{{\n * // The following selects people with age 15, 20, or 30.\n * df.select( df('age).isin(15, 20, 30) )\n * }}}\n *\n * @param values are constants of the same type\n * apache/spark\n */\n def isin(values: U*)(implicit e: CatalystIsin[U]): ThisType[T, Boolean] =\n typed(self.untyped.isin(values:_*))\n\n /**\n * True if the current column is between the lower bound and upper bound, inclusive.\n *\n * @param lowerBound a constant of the same type\n * @param upperBound a constant of the same type\n * apache/spark\n */\n def between(lowerBound: U, upperBound: U)(implicit i0: CatalystOrdered[U]): ThisType[T, Boolean] =\n typed(self.untyped.between(lit(lowerBound)(self.uencoder).untyped, lit(upperBound)(self.uencoder).untyped))\n\n /**\n * True if the current column is between the lower bound and upper bound, inclusive.\n *\n * @param lowerBound another column of the same type\n * @param upperBound another column of the same type\n * apache/spark\n */\n def between[TT1, TT2, W1, W2](lowerBound: ThisType[TT1, U], upperBound: ThisType[TT2, U])\n (implicit\n i0: CatalystOrdered[U],\n w0: With.Aux[T, TT1, W1],\n w1: With.Aux[TT2, W1, W2]\n ): ThisType[W2, Boolean] =\n typed(self.untyped.between(lowerBound.untyped, upperBound.untyped))\n\n /**\n * Returns a nested column matching the field `symbol`.\n * \n * @param symbol the field symbol\n * @tparam V the type of the nested field\n */\n def field[V](symbol: Witness.Lt[Symbol])(implicit\n i0: TypedColumn.Exists[U, symbol.T, V],\n i1: TypedEncoder[V]\n ): ThisType[T, V] = \n typed(self.untyped.getField(symbol.value.name))\n\n}\n\n\nsealed class SortedTypedColumn[T, U](val expr: Expression)(\n implicit\n val uencoder: TypedEncoder[U]\n) extends UntypedExpression[T] {\n\n def this(column: Column)(implicit e: TypedEncoder[U]) = {\n this(FramelessInternals.expr(column))\n }\n\n def untyped: Column = new Column(expr)\n}\n\nobject SortedTypedColumn {\n implicit def defaultAscending[T, U : CatalystOrdered](typedColumn: TypedColumn[T, U]): SortedTypedColumn[T, U] =\n new SortedTypedColumn[T, U](typedColumn.untyped.asc)(typedColumn.uencoder)\n\n object defaultAscendingPoly extends Poly1 {\n implicit def caseTypedColumn[T, U : CatalystOrdered] = at[TypedColumn[T, U]](c => defaultAscending(c))\n implicit def caseTypeSortedColumn[T, U] = at[SortedTypedColumn[T, U]](identity)\n }\n}\n\nobject TypedColumn {\n /** Evidence that type `T` has column `K` with type `V`. */\n @implicitNotFound(msg = \"No column ${K} of type ${V} in ${T}\")\n trait Exists[T, K, V]\n\n @implicitNotFound(msg = \"No columns ${K} of type ${V} in ${T}\")\n trait ExistsMany[T, K <: HList, V]\n\n object ExistsMany {\n implicit def deriveCons[T, KH, KT <: HList, V0, V1]\n (implicit\n head: Exists[T, KH, V0],\n tail: ExistsMany[V0, KT, V1]\n ): ExistsMany[T, KH :: KT, V1] =\n new ExistsMany[T, KH :: KT, V1] {}\n\n implicit def deriveHNil[T, K, V](implicit head: Exists[T, K, V]): ExistsMany[T, K :: HNil, V] =\n new ExistsMany[T, K :: HNil, V] {}\n }\n\n object Exists {\n def apply[T, V](column: Witness)(implicit e: Exists[T, column.T, V]): Exists[T, column.T, V] = e\n\n implicit def deriveRecord[T, H <: HList, K, V]\n (implicit\n i0: LabelledGeneric.Aux[T, H],\n i1: Selector.Aux[H, K, V]\n ): Exists[T, K, V] = new Exists[T, K, V] {}\n }\n\n /**\n * {{{\n * import frameless.TypedColumn\n * \n * case class Foo(id: Int, bar: String)\n * \n * val colbar: TypedColumn[Foo, String] = TypedColumn { foo: Foo => foo.bar }\n * val colid = TypedColumn[Foo, Int](_.id)\n * }}}\n */\n def apply[T, U](x: T => U): TypedColumn[T, U] =\n macro TypedColumnMacroImpl.applyImpl[T, U]\n\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedColumnMacroImpl.scala", "content": "package frameless\n\nimport scala.reflect.macros.whitebox\n\nprivate[frameless] object TypedColumnMacroImpl {\n\n def applyImpl[T: c.WeakTypeTag, U: c.WeakTypeTag](c: whitebox.Context)(x: c.Tree): c.Expr[TypedColumn[T, U]] = {\n import c.universe._\n\n val t = c.weakTypeOf[T]\n val u = c.weakTypeOf[U]\n\n def buildExpression(path: List[String]): c.Expr[TypedColumn[T, U]] = {\n val columnName = path.mkString(\".\")\n\n c.Expr[TypedColumn[T, U]](q\"new _root_.frameless.TypedColumn[$t, $u]((org.apache.spark.sql.functions.col($columnName)).expr)\")\n }\n\n def abort(msg: String) = c.abort(c.enclosingPosition, msg)\n\n @annotation.tailrec\n def path(in: Select, out: List[TermName]): List[TermName] =\n in.qualifier match {\n case sub: Select =>\n path(sub, in.name.toTermName :: out)\n\n case id: Ident =>\n id.name.toTermName :: in.name.toTermName :: out\n\n case u =>\n abort(s\"Unsupported selection: $u\")\n }\n\n @annotation.tailrec\n def check(current: Type, in: List[TermName]): Boolean = in match {\n case next :: tail => {\n val sym = current.decl(next).asTerm\n\n if (!sym.isStable) {\n abort(s\"Stable term expected: ${current}.${next}\")\n }\n\n check(sym.info, tail)\n }\n\n case _ =>\n true\n }\n\n x match {\n case fn: Function => fn.body match {\n case select: Select if select.name.isTermName =>\n val expectedRoot: Option[String] = fn.vparams match {\n case List(rt) if rt.rhs == EmptyTree =>\n Option.empty[String]\n\n case List(rt) =>\n Some(rt.toString)\n\n case u =>\n abort(s\"Select expression must have a single parameter: ${u mkString \", \"}\")\n }\n\n path(select, List.empty) match {\n case root :: tail if (\n expectedRoot.forall(_ == root) && check(t, tail)) => {\n val colPath = tail.mkString(\".\")\n\n c.Expr[TypedColumn[T, U]](q\"new _root_.frameless.TypedColumn[$t, $u]((org.apache.spark.sql.functions.col($colPath)).expr)\")\n }\n\n case _ =>\n abort(s\"Invalid select expression: $select\")\n }\n\n case t =>\n abort(s\"Select expression expected: $t\")\n }\n\n case _ =>\n abort(s\"Function expected: $x\")\n }\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedDataset.scala", "content": "package frameless\n\nimport java.util\nimport frameless.functions.CatalystExplodableCollection\nimport frameless.ops._\nimport org.apache.spark.rdd.RDD\nimport org.apache.spark.sql.{Column, DataFrame, Dataset, FramelessInternals, SparkSession}\nimport org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Literal}\nimport org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint}\nimport org.apache.spark.sql.catalyst.plans.Inner\nimport org.apache.spark.sql.types.StructType\nimport shapeless._\nimport shapeless.labelled.FieldType\nimport shapeless.ops.hlist.{Diff, IsHCons, Mapper, Prepend, ToTraversable, Tupler}\nimport shapeless.ops.record.{Keys, Modifier, Remover, Values}\n\nimport scala.language.experimental.macros\n\n/** [[TypedDataset]] is a safer interface for working with `Dataset`.\n *\n * NOTE: Prefer `TypedDataset.create` over `new TypedDataset` unless you\n * know what you are doing.\n *\n * Documentation marked \"apache/spark\" is thanks to apache/spark Contributors\n * at https://github.com/apache/spark, licensed under Apache v2.0 available at\n * http://www.apache.org/licenses/LICENSE-2.0\n */\nclass TypedDataset[T] protected[frameless](val dataset: Dataset[T])(implicit val encoder: TypedEncoder[T])\n extends TypedDatasetForwarded[T] { self =>\n\n private implicit val spark: SparkSession = dataset.sparkSession\n\n /** Aggregates on the entire Dataset without groups.\n *\n * apache/spark\n */\n def agg[A](ca: TypedAggregate[T, A]): TypedDataset[A] = {\n implicit val ea = ca.uencoder\n val tuple1: TypedDataset[Tuple1[A]] = aggMany(ca)\n\n // now we need to unpack `Tuple1[A]` to `A`\n TypedEncoder[A].catalystRepr match {\n case StructType(_) =>\n // if column is struct, we use all its fields\n val df = tuple1\n .dataset\n .selectExpr(\"_1.*\")\n .as[A](TypedExpressionEncoder[A])\n\n TypedDataset.create(df)\n case other =>\n // for primitive types `Tuple1[A]` has the same schema as `A`\n TypedDataset.create(tuple1.dataset.as[A](TypedExpressionEncoder[A]))\n }\n }\n\n /** Aggregates on the entire Dataset without groups.\n *\n * apache/spark\n */\n def agg[A, B](\n ca: TypedAggregate[T, A],\n cb: TypedAggregate[T, B]\n ): TypedDataset[(A, B)] = {\n implicit val (ea, eb) = (ca.uencoder, cb.uencoder)\n aggMany(ca, cb)\n }\n\n /** Aggregates on the entire Dataset without groups.\n *\n * apache/spark\n */\n def agg[A, B, C](\n ca: TypedAggregate[T, A],\n cb: TypedAggregate[T, B],\n cc: TypedAggregate[T, C]\n ): TypedDataset[(A, B, C)] = {\n implicit val (ea, eb, ec) = (ca.uencoder, cb.uencoder, cc.uencoder)\n aggMany(ca, cb, cc)\n }\n\n /** Aggregates on the entire Dataset without groups.\n *\n * apache/spark\n */\n def agg[A, B, C, D](\n ca: TypedAggregate[T, A],\n cb: TypedAggregate[T, B],\n cc: TypedAggregate[T, C],\n cd: TypedAggregate[T, D]\n ): TypedDataset[(A, B, C, D)] = {\n implicit val (ea, eb, ec, ed) = (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder)\n aggMany(ca, cb, cc, cd)\n }\n\n /** Aggregates on the entire Dataset without groups.\n *\n * apache/spark\n */\n object aggMany extends ProductArgs {\n def applyProduct[U <: HList, Out0 <: HList, Out](columns: U)\n (implicit\n i0: AggregateTypes.Aux[T, U, Out0],\n i1: ToTraversable.Aux[U, List, UntypedExpression[T]],\n i2: Tupler.Aux[Out0, Out],\n i3: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n\n val underlyingColumns = columns.toList[UntypedExpression[T]]\n val cols: Seq[Column] = for {\n (c, i) <- columns.toList[UntypedExpression[T]].zipWithIndex\n } yield new Column(c.expr).as(s\"_${i+1}\")\n\n // Workaround to SPARK-20346. One alternative is to allow the result to be Vector(null) for empty DataFrames.\n // Another one would be to return an Option.\n val filterStr = (\n for {\n (c, i) <- underlyingColumns.zipWithIndex\n if !c.uencoder.nullable\n } yield s\"_${i+1} is not null\"\n ).mkString(\" or \")\n\n val selected = dataset.toDF().agg(cols.head, cols.tail:_*).as[Out](TypedExpressionEncoder[Out])\n TypedDataset.create[Out](if (filterStr.isEmpty) selected else selected.filter(filterStr))\n }\n }\n\n /** Returns a new [[TypedDataset]] where each record has been mapped on to the specified type. */\n def as[U]()(implicit as: As[T, U]): TypedDataset[U] = {\n implicit val uencoder = as.encoder\n TypedDataset.create(dataset.as[U](TypedExpressionEncoder[U]))\n }\n\n /** Returns a checkpointed version of this [[TypedDataset]]. Checkpointing can be used to truncate the\n * logical plan of this Dataset, which is especially useful in iterative algorithms where the\n * plan may grow exponentially. It will be saved to files inside the checkpoint\n * directory set with `SparkContext#setCheckpointDir`.\n *\n * Differs from `Dataset#checkpoint` by wrapping its result into an effect-suspending `F[_]`.\n *\n * apache/spark\n */\n def checkpoint[F[_]](eager: Boolean)(implicit F: SparkDelay[F]): F[TypedDataset[T]] =\n F.delay(TypedDataset.create[T](dataset.checkpoint(eager)))\n\n /** Returns a new [[TypedDataset]] where each record has been mapped on to the specified type.\n * Unlike `as` the projection U may include a subset of the columns of T and the column names and types must agree.\n *\n * {{{\n * case class Foo(i: Int, j: String)\n * case class Bar(j: String)\n *\n * val t: TypedDataset[Foo] = ...\n * val b: TypedDataset[Bar] = t.project[Bar]\n *\n * case class BarErr(e: String)\n * // The following does not compile because `Foo` doesn't have a field with name `e`\n * val e: TypedDataset[BarErr] = t.project[BarErr]\n * }}}\n */\n def project[U](implicit projector: SmartProject[T,U]): TypedDataset[U] = projector.apply(this)\n\n /** Returns a new [[TypedDataset]] that contains the elements of both this and the `other` [[TypedDataset]]\n * combined.\n *\n * Note that, this function is not a typical set union operation, in that it does not eliminate\n * duplicate items. As such, it is analogous to `UNION ALL` in SQL.\n *\n * Differs from `Dataset#union` by aligning fields if possible.\n * It will not compile if `Datasets` have not compatible schema.\n *\n * Example:\n * {{{\n * case class Foo(x: Int, y: Long)\n * case class Bar(y: Long, x: Int)\n * case class Faz(x: Int, y: Int, z: Int)\n *\n * foo: TypedDataset[Foo] = ...\n * bar: TypedDataset[Bar] = ...\n * faz: TypedDataset[Faz] = ...\n *\n * foo union bar: TypedDataset[Foo]\n * foo union faz: TypedDataset[Foo]\n * // won't compile, you need to reverse order, you can't project from less fields to more\n * faz union foo\n *\n * }}}\n *\n * apache/spark\n */\n def union[U: TypedEncoder](other: TypedDataset[U])(implicit projector: SmartProject[U, T]): TypedDataset[T] =\n TypedDataset.create(dataset.union(other.project[T].dataset))\n\n /** Returns a new [[TypedDataset]] that contains the elements of both this and the `other` [[TypedDataset]]\n * combined.\n *\n * Note that, this function is not a typical set union operation, in that it does not eliminate\n * duplicate items. As such, it is analogous to `UNION ALL` in SQL.\n *\n * apache/spark\n */\n def union(other: TypedDataset[T]): TypedDataset[T] = {\n TypedDataset.create(dataset.union(other.dataset))\n }\n\n /** Returns the number of elements in the [[TypedDataset]].\n *\n * Differs from `Dataset#count` by wrapping its result into an effect-suspending `F[_]`.\n */\n def count[F[_]]()(implicit F: SparkDelay[F]): F[Long] =\n F.delay(dataset.count())\n\n /** Returns `TypedColumn` of type `A` given its name (alias for `col`).\n *\n * {{{\n * tf('id)\n * }}}\n *\n * It is statically checked that column with such name exists and has type `A`.\n */\n def apply[A](column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, A],\n i1: TypedEncoder[A]\n ): TypedColumn[T, A] = col(column)\n\n /** Returns `TypedColumn` of type `A` given its name.\n *\n * {{{\n * tf.col('id)\n * }}}\n *\n * It is statically checked that column with such name exists and has type `A`.\n */\n def col[A](column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, A],\n i1: TypedEncoder[A]\n ): TypedColumn[T, A] =\n new TypedColumn[T, A](dataset(column.value.name).as[A](TypedExpressionEncoder[A]))\n\n /** Returns `TypedColumn` of type `A` given a lambda indicating the field.\n *\n * {{{\n * td.col(_.id)\n * }}}\n *\n * It is statically checked that column with such name exists and has type `A`.\n */\n def col[A](x: Function1[T, A]): TypedColumn[T, A] =\n macro TypedColumnMacroImpl.applyImpl[T, A]\n\n /** Projects the entire `TypedDataset[T]` into a single column of type `TypedColumn[T,T]`.\n * {{{\n * ts: TypedDataset[Foo] = ...\n * ts.select(ts.asCol, ts.asCol): TypedDataset[(Foo,Foo)]\n * }}}\n */\n def asCol: TypedColumn[T, T] = {\n val projectedColumn: Column = encoder.catalystRepr match {\n case StructType(_) =>\n val allColumns: Array[Column] = dataset.columns.map(dataset.col)\n org.apache.spark.sql.functions.struct(allColumns.toSeq: _*)\n\n case _ =>\n dataset.col(dataset.columns.head)\n }\n \n new TypedColumn[T,T](projectedColumn)\n }\n\n /** References the entire `TypedDataset[T]` as a single column \n * of type `TypedColumn[T,T]` so it can be used in a join operation.\n * \n * {{{\n * def nameJoin(ds1: TypedDataset[Person], ds2: TypedDataset[Name]) =\n * ds1.joinLeftSemi(ds2)(ds1.col('name) === ds2.asJoinColValue)\n * }}}\n */\n def asJoinColValue(implicit i0: IsValueClass[T]): TypedColumn[T, T] = {\n import _root_.frameless.syntax._\n\n dataset.col(\"value\").typedColumn\n }\n\n object colMany extends SingletonProductArgs {\n def applyProduct[U <: HList, Out](columns: U)\n (implicit\n i0: TypedColumn.ExistsMany[T, U, Out],\n i1: TypedEncoder[Out],\n i2: ToTraversable.Aux[U, List, Symbol]\n ): TypedColumn[T, Out] = {\n val names = columns.toList[Symbol].map(_.name)\n val colExpr = FramelessInternals.resolveExpr(dataset, names)\n new TypedColumn[T, Out](colExpr)\n }\n }\n\n /** Right hand side disambiguation of `col` for join expressions.\n * To be used when writting self-joins, noop in other circumstances.\n *\n * Note: In vanilla Spark, disambiguation in self-joins is acheaved using\n * String based aliases, which is obviously unsafe.\n */\n def colRight[A](column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, A],\n i1: TypedEncoder[A]\n ): TypedColumn[T, A] =\n new TypedColumn[T, A](FramelessInternals.DisambiguateRight(col(column).expr))\n\n /** Left hand side disambiguation of `col` for join expressions.\n * To be used when writting self-joins, noop in other circumstances.\n *\n * Note: In vanilla Spark, disambiguation in self-joins is acheaved using\n * String based aliases, which is obviously unsafe.\n */\n def colLeft[A](column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, A],\n i1: TypedEncoder[A]\n ): TypedColumn[T, A] =\n new TypedColumn[T, A](FramelessInternals.DisambiguateLeft(col(column).expr))\n\n /** Returns a `Seq` that contains all the elements in this [[TypedDataset]].\n *\n * Running this operation requires moving all the data into the application's driver process, and\n * doing so on a very large [[TypedDataset]] can crash the driver process with OutOfMemoryError.\n *\n * Differs from `Dataset#collect` by wrapping its result into an effect-suspending `F[_]`.\n */\n def collect[F[_]]()(implicit F: SparkDelay[F]): F[Seq[T]] =\n F.delay(dataset.collect().toSeq)\n\n /** Optionally returns the first element in this [[TypedDataset]].\n *\n * Differs from `Dataset#first` by wrapping its result into an `Option` and an effect-suspending `F[_]`.\n */\n def firstOption[F[_]]()(implicit F: SparkDelay[F]): F[Option[T]] =\n F.delay {\n try {\n Option(dataset.first())\n } catch {\n case e: NoSuchElementException => None\n }\n }\n\n /** Returns the first `num` elements of this [[TypedDataset]] as a `Seq`.\n *\n * Running take requires moving data into the application's driver process, and doing so with\n * a very large `num` can crash the driver process with OutOfMemoryError.\n *\n * Differs from `Dataset#take` by wrapping its result into an effect-suspending `F[_]`.\n *\n * apache/spark\n */\n def take[F[_]](num: Int)(implicit F: SparkDelay[F]): F[Seq[T]] =\n F.delay(dataset.take(num).toSeq)\n\n /** Return an iterator that contains all rows in this [[TypedDataset]].\n *\n * The iterator will consume as much memory as the largest partition in this [[TypedDataset]].\n *\n * NOTE: this results in multiple Spark jobs, and if the input [[TypedDataset]] is the result\n * of a wide transformation (e.g. join with different partitioners), to avoid\n * recomputing the input [[TypedDataset]] should be cached first.\n *\n * Differs from `Dataset#toLocalIterator()` by wrapping its result into an effect-suspending `F[_]`.\n *\n * apache/spark\n */\n def toLocalIterator[F[_]]()(implicit F: SparkDelay[F]): F[util.Iterator[T]] =\n F.delay(dataset.toLocalIterator())\n\n /** Alias for firstOption().\n */\n def headOption[F[_]]()(implicit F: SparkDelay[F]): F[Option[T]] = firstOption()\n\n /** Alias for take().\n */\n def head[F[_]](num: Int)(implicit F: SparkDelay[F]): F[Seq[T]] = take(num)\n\n // $COVERAGE-OFF$\n /** Alias for firstOption().\n */\n @deprecated(\"Method may throw exception. Use headOption or firstOption instead.\", \"0.5.0\")\n def head: T = dataset.head()\n\n /** Alias for firstOption().\n */\n @deprecated(\"Method may throw exception. Use headOption or firstOption instead.\", \"0.5.0\")\n def first: T = dataset.head()\n // $COVERAGE-ONN$\n\n /** Displays the content of this [[TypedDataset]] in a tabular form. Strings more than 20 characters\n * will be truncated, and all cells will be aligned right. For example:\n * {{{\n * year month AVG('Adj Close) MAX('Adj Close)\n * 1980 12 0.503218 0.595103\n * 1981 01 0.523289 0.570307\n * 1982 02 0.436504 0.475256\n * 1983 03 0.410516 0.442194\n * 1984 04 0.450090 0.483521\n * }}}\n * @param numRows Number of rows to show\n * @param truncate Whether truncate long strings. If true, strings more than 20 characters will\n * be truncated and all cells will be aligned right\n *\n * Differs from `Dataset#show` by wrapping its result into an effect-suspending `F[_]`.\n *\n * apache/spark\n */\n def show[F[_]](numRows: Int = 20, truncate: Boolean = true)(implicit F: SparkDelay[F]): F[Unit] =\n F.delay(dataset.show(numRows, truncate))\n\n /** Returns a new [[frameless.TypedDataset]] that only contains elements where `column` is `true`.\n *\n * Differs from `TypedDatasetForward#filter` by taking a `TypedColumn[T, Boolean]` instead of a\n * `T => Boolean`. Using a column expression instead of a regular function save one Spark → Scala\n * deserialization which leads to better performance.\n */\n def filter(column: TypedColumn[T, Boolean]): TypedDataset[T] = {\n val filtered = dataset.toDF()\n .filter(column.untyped)\n .as[T](TypedExpressionEncoder[T])\n\n TypedDataset.create[T](filtered)\n }\n\n /** Runs `func` on each element of this [[TypedDataset]].\n *\n * Differs from `Dataset#foreach` by wrapping its result into an effect-suspending `F[_]`.\n */\n def foreach[F[_]](func: T => Unit)(implicit F: SparkDelay[F]): F[Unit] =\n F.delay(dataset.foreach(func))\n\n /** Runs `func` on each partition of this [[TypedDataset]].\n *\n * Differs from `Dataset#foreachPartition` by wrapping its result into an effect-suspending `F[_]`.\n */\n def foreachPartition[F[_]](func: Iterator[T] => Unit)(implicit F: SparkDelay[F]): F[Unit] =\n F.delay(dataset.foreachPartition(func))\n\n /**\n * Create a multi-dimensional cube for the current [[TypedDataset]] using the specified column,\n * so we can run aggregation on it.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * Differs from `Dataset#cube` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n def cube[K1](\n c1: TypedColumn[T, K1]\n ): Cube1Ops[K1, T] = new Cube1Ops[K1, T](this, c1)\n\n /**\n * Create a multi-dimensional cube for the current [[TypedDataset]] using the specified columns,\n * so we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * Differs from `Dataset#cube` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n def cube[K1, K2](\n c1: TypedColumn[T, K1],\n c2: TypedColumn[T, K2]\n ): Cube2Ops[K1, K2, T] = new Cube2Ops[K1, K2, T](this, c1, c2)\n\n /**\n * Create a multi-dimensional cube for the current [[TypedDataset]] using the specified columns,\n * so we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * {{{\n * case class MyClass(a: Int, b: Int, c: Int)\n * val ds: TypedDataset[MyClass]\n\n * val cubeDataset: TypedDataset[(Option[A], Option[B], Long)] =\n * ds.cubeMany(ds('a), ds('b)).agg(count[MyClass]())\n *\n * // original dataset:\n * a b c\n * 10 20 1\n * 15 25 2\n *\n * // after aggregation:\n * _1 _2 _3\n * 15 null 1\n * 15 25 1\n * null null 2\n * null 25 1\n * null 20 1\n * 10 null 1\n * 10 20 1\n *\n * }}}\n *\n * Differs from `Dataset#cube` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n object cubeMany extends ProductArgs {\n def applyProduct[TK <: HList, K <: HList, KT](groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: Tupler.Aux[K, KT],\n i2: ToTraversable.Aux[TK, List, UntypedExpression[T]]\n ): CubeManyOps[T, TK, K, KT] = new CubeManyOps[T, TK, K, KT](self, groupedBy)\n }\n\n /**\n * Groups the [[TypedDataset]] using the specified columns, so that we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * apache/spark\n */\n def groupBy[K1](\n c1: TypedColumn[T, K1]\n ): GroupedBy1Ops[K1, T] = new GroupedBy1Ops[K1, T](this, c1)\n\n /**\n * Groups the [[TypedDataset]] using the specified columns, so that we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * apache/spark\n */\n def groupBy[K1, K2](\n c1: TypedColumn[T, K1],\n c2: TypedColumn[T, K2]\n ): GroupedBy2Ops[K1, K2, T] = new GroupedBy2Ops[K1, K2, T](this, c1, c2)\n\n /**\n * Groups the [[TypedDataset]] using the specified columns, so that we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * {{{\n * case class MyClass(a: Int, b: Int, c: Int)\n * val ds: TypedDataset[MyClass]\n *\n * val cubeDataset: TypedDataset[(Option[A], Option[B], Long)] =\n * ds.groupByMany(ds('a), ds('b)).agg(count[MyClass]())\n *\n * // original dataset:\n * a b c\n * 10 20 1\n * 15 25 2\n *\n * // after aggregation:\n * _1 _2 _3\n * 10 20 1\n * 15 25 1\n *\n * }}}\n *\n * apache/spark\n */\n object groupByMany extends ProductArgs {\n def applyProduct[TK <: HList, K <: HList, KT](groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: Tupler.Aux[K, KT],\n i2: ToTraversable.Aux[TK, List, UntypedExpression[T]]\n ): GroupedByManyOps[T, TK, K, KT] = new GroupedByManyOps[T, TK, K, KT](self, groupedBy)\n }\n\n /**\n * Create a multi-dimensional rollup for the current [[TypedDataset]] using the specified column,\n * so we can run aggregation on it.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * Differs from `Dataset#rollup` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n def rollup[K1](\n c1: TypedColumn[T, K1]\n ): Rollup1Ops[K1, T] = new Rollup1Ops[K1, T](this, c1)\n\n /**\n * Create a multi-dimensional rollup for the current [[TypedDataset]] using the specified columns,\n * so we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * Differs from `Dataset#rollup` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n def rollup[K1, K2](\n c1: TypedColumn[T, K1],\n c2: TypedColumn[T, K2]\n ): Rollup2Ops[K1, K2, T] = new Rollup2Ops[K1, K2, T](this, c1, c2)\n\n /**\n * Create a multi-dimensional rollup for the current [[TypedDataset]] using the specified columns,\n * so we can run aggregation on them.\n * See [[frameless.functions.AggregateFunctions]] for all the available aggregate functions.\n *\n * {{{\n * case class MyClass(a: Int, b: Int, c: Int)\n * val ds: TypedDataset[MyClass]\n *\n * val cubeDataset: TypedDataset[(Option[A], Option[B], Long)] =\n * ds.rollupMany(ds('a), ds('b)).agg(count[MyClass]())\n *\n * // original dataset:\n * a b c\n * 10 20 1\n * 15 25 2\n *\n * // after aggregation:\n * _1 _2 _3\n * 15 null 1\n * 15 25 1\n * null null 2\n * 10 null 1\n * 10 20 1\n *\n * }}}\n *\n * Differs from `Dataset#rollup` by wrapping values into `Option` instead of returning `null`.\n *\n * apache/spark\n */\n object rollupMany extends ProductArgs {\n def applyProduct[TK <: HList, K <: HList, KT](groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: Tupler.Aux[K, KT],\n i2: ToTraversable.Aux[TK, List, UntypedExpression[T]]\n ): RollupManyOps[T, TK, K, KT] = new RollupManyOps[T, TK, K, KT](self, groupedBy)\n }\n\n /** Computes the cartesian project of `this` `Dataset` with the `other` `Dataset` */\n def joinCross[U](other: TypedDataset[U])\n (implicit e: TypedEncoder[(T, U)]): TypedDataset[(T, U)] =\n new TypedDataset(self.dataset.joinWith(other.dataset, new Column(Literal(true)), \"cross\"))\n\n /** Computes the full outer join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinFull[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean])\n (implicit e: TypedEncoder[(Option[T], Option[U])]): TypedDataset[(Option[T], Option[U])] =\n new TypedDataset(self.dataset.joinWith(other.dataset, condition.untyped, \"full\")\n .as[(Option[T], Option[U])](TypedExpressionEncoder[(Option[T], Option[U])]))\n\n /** Computes the inner join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinInner[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean])\n (implicit e: TypedEncoder[(T, U)]): TypedDataset[(T, U)] = {\n import FramelessInternals._\n\n val leftPlan = logicalPlan(dataset)\n val rightPlan = logicalPlan(other.dataset)\n val join = disambiguate(Join(leftPlan, rightPlan, Inner, Some(condition.expr), JoinHint.NONE))\n val joinedPlan = joinPlan(dataset, join, leftPlan, rightPlan)\n val joinedDs = mkDataset(dataset.sqlContext, joinedPlan, TypedExpressionEncoder[(T, U)])\n\n TypedDataset.create[(T, U)](joinedDs)\n }\n\n /** Computes the left outer join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinLeft[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean])\n (implicit e: TypedEncoder[(T, Option[U])]): TypedDataset[(T, Option[U])] =\n new TypedDataset(self.dataset.joinWith(other.dataset, condition.untyped, \"left_outer\")\n .as[(T, Option[U])](TypedExpressionEncoder[(T, Option[U])]))\n\n /** Computes the left semi join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinLeftSemi[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean]): TypedDataset[T] =\n new TypedDataset(self.dataset.join(other.dataset, condition.untyped, \"leftsemi\")\n .as[T](TypedExpressionEncoder(encoder)))\n\n /** Computes the left anti join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinLeftAnti[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean]): TypedDataset[T] =\n new TypedDataset(self.dataset.join(other.dataset, condition.untyped, \"leftanti\")\n .as[T](TypedExpressionEncoder(encoder)))\n\n /** Computes the right outer join of `this` `Dataset` with the `other` `Dataset`,\n * returning a `Tuple2` for each pair where condition evaluates to true.\n */\n def joinRight[U](other: TypedDataset[U])(condition: TypedColumn[T with U, Boolean])\n (implicit e: TypedEncoder[(Option[T], U)]): TypedDataset[(Option[T], U)] =\n new TypedDataset(self.dataset.joinWith(other.dataset, condition.untyped, \"right_outer\")\n .as[(Option[T], U)](TypedExpressionEncoder[(Option[T], U)]))\n\n private def disambiguate(join: Join): Join = {\n val plan = FramelessInternals.ofRows(dataset.sparkSession, join).queryExecution.analyzed.asInstanceOf[Join]\n val disambiguated = plan.condition.map(_.transform {\n case FramelessInternals.DisambiguateLeft(tagged: AttributeReference) =>\n val leftDs = FramelessInternals.ofRows(spark, plan.left)\n FramelessInternals.resolveExpr(leftDs, Seq(tagged.name))\n\n case FramelessInternals.DisambiguateRight(tagged: AttributeReference) =>\n val rightDs = FramelessInternals.ofRows(spark, plan.right)\n FramelessInternals.resolveExpr(rightDs, Seq(tagged.name))\n\n case x => x\n })\n plan.copy(condition = disambiguated)\n }\n\n /** Takes a function from A => R and converts it to a UDF for TypedColumn[T, A] => TypedColumn[T, R].\n */\n def makeUDF[A: TypedEncoder, R: TypedEncoder](f: A => R):\n TypedColumn[T, A] => TypedColumn[T, R] = functions.udf(f)\n\n /** Takes a function from (A1, A2) => R and converts it to a UDF for\n * (TypedColumn[T, A1], TypedColumn[T, A2]) => TypedColumn[T, R].\n */\n def makeUDF[A1: TypedEncoder, A2: TypedEncoder, R: TypedEncoder](f: (A1, A2) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2]) => TypedColumn[T, R] = functions.udf(f)\n\n /** Takes a function from (A1, A2, A3) => R and converts it to a UDF for\n * (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3]) => TypedColumn[T, R].\n */\n def makeUDF[A1: TypedEncoder, A2: TypedEncoder, A3: TypedEncoder, R: TypedEncoder](f: (A1, A2, A3) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3]) => TypedColumn[T, R] = functions.udf(f)\n\n /** Takes a function from (A1, A2, A3, A4) => R and converts it to a UDF for\n * (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4]) => TypedColumn[T, R].\n */\n def makeUDF[A1: TypedEncoder, A2: TypedEncoder, A3: TypedEncoder, A4: TypedEncoder, R: TypedEncoder](f: (A1, A2, A3, A4) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4]) => TypedColumn[T, R] = functions.udf(f)\n\n /** Takes a function from (A1, A2, A3, A4, A5) => R and converts it to a UDF for\n * (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4], TypedColumn[T, A5]) => TypedColumn[T, R].\n */\n def makeUDF[A1: TypedEncoder, A2: TypedEncoder, A3: TypedEncoder, A4: TypedEncoder, A5: TypedEncoder, R: TypedEncoder](f: (A1, A2, A3, A4, A5) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4], TypedColumn[T, A5]) => TypedColumn[T, R] = functions.udf(f)\n\n /** Type-safe projection from type T to Tuple1[A]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A](\n ca: TypedColumn[T, A]\n ): TypedDataset[A] = {\n implicit val ea = ca.uencoder\n\n val tuple1: TypedDataset[Tuple1[A]] = selectMany(ca)\n\n // now we need to unpack `Tuple1[A]` to `A`\n\n TypedEncoder[A].catalystRepr match {\n case StructType(_) =>\n // if column is struct, we use all its fields\n val df = tuple1\n .dataset\n .selectExpr(\"_1.*\")\n .as[A](TypedExpressionEncoder[A])\n\n TypedDataset.create(df)\n case other =>\n // for primitive types `Tuple1[A]` has the same schema as `A`\n TypedDataset.create(tuple1.dataset.as[A](TypedExpressionEncoder[A]))\n }\n }\n\n /** Type-safe projection from type T to Tuple2[A,B]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B]\n ): TypedDataset[(A, B)] = {\n implicit val (ea, eb) = (ca.uencoder, cb.uencoder)\n selectMany(ca, cb)\n }\n\n /** Type-safe projection from type T to Tuple3[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C]\n ): TypedDataset[(A, B, C)] = {\n implicit val (ea, eb, ec) = (ca.uencoder, cb.uencoder, cc.uencoder)\n selectMany(ca, cb, cc)\n }\n\n /** Type-safe projection from type T to Tuple4[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D]\n ): TypedDataset[(A, B, C, D)] = {\n implicit val (ea, eb, ec, ed) = (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder)\n selectMany(ca, cb, cc, cd)\n }\n\n /** Type-safe projection from type T to Tuple5[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E]\n ): TypedDataset[(A, B, C, D, E)] = {\n implicit val (ea, eb, ec, ed, ee) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder)\n\n selectMany(ca, cb, cc, cd, ce)\n }\n\n /** Type-safe projection from type T to Tuple6[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E, F](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E],\n cf: TypedColumn[T, F]\n ): TypedDataset[(A, B, C, D, E, F)] = {\n implicit val (ea, eb, ec, ed, ee, ef) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder, cf.uencoder)\n\n selectMany(ca, cb, cc, cd, ce, cf)\n }\n\n /** Type-safe projection from type T to Tuple7[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E, F, G](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E],\n cf: TypedColumn[T, F],\n cg: TypedColumn[T, G]\n ): TypedDataset[(A, B, C, D, E, F, G)] = {\n implicit val (ea, eb, ec, ed, ee, ef, eg) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder, cf.uencoder, cg.uencoder)\n\n selectMany(ca, cb, cc, cd, ce, cf, cg)\n }\n\n /** Type-safe projection from type T to Tuple8[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E, F, G, H](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E],\n cf: TypedColumn[T, F],\n cg: TypedColumn[T, G],\n ch: TypedColumn[T, H]\n ): TypedDataset[(A, B, C, D, E, F, G, H)] = {\n implicit val (ea, eb, ec, ed, ee, ef, eg, eh) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder, cf.uencoder, cg.uencoder, ch.uencoder)\n\n selectMany(ca, cb, cc, cd, ce, cf, cg, ch)\n }\n\n /** Type-safe projection from type T to Tuple9[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E, F, G, H, I](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E],\n cf: TypedColumn[T, F],\n cg: TypedColumn[T, G],\n ch: TypedColumn[T, H],\n ci: TypedColumn[T, I]\n ): TypedDataset[(A, B, C, D, E, F, G, H, I)] = {\n implicit val (ea, eb, ec, ed, ee, ef, eg, eh, ei) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder, cf.uencoder, cg.uencoder, ch.uencoder, ci.uencoder)\n\n selectMany(ca, cb, cc, cd, ce, cf, cg, ch, ci)\n }\n\n /** Type-safe projection from type T to Tuple10[A,B,...]\n * {{{\n * d.select( d('a), d('a)+d('b), ... )\n * }}}\n */\n def select[A, B, C, D, E, F, G, H, I, J](\n ca: TypedColumn[T, A],\n cb: TypedColumn[T, B],\n cc: TypedColumn[T, C],\n cd: TypedColumn[T, D],\n ce: TypedColumn[T, E],\n cf: TypedColumn[T, F],\n cg: TypedColumn[T, G],\n ch: TypedColumn[T, H],\n ci: TypedColumn[T, I],\n cj: TypedColumn[T, J]\n ): TypedDataset[(A, B, C, D, E, F, G, H, I, J)] = {\n implicit val (ea, eb, ec, ed, ee, ef, eg, eh, ei, ej) =\n (ca.uencoder, cb.uencoder, cc.uencoder, cd.uencoder, ce.uencoder, cf.uencoder, cg.uencoder, ch.uencoder, ci.uencoder, cj.uencoder)\n selectMany(ca, cb, cc, cd, ce, cf, cg, ch, ci, cj)\n }\n\n object selectMany extends ProductArgs {\n def applyProduct[U <: HList, Out0 <: HList, Out](columns: U)\n (implicit\n i0: ColumnTypes.Aux[T, U, Out0],\n i1: ToTraversable.Aux[U, List, UntypedExpression[T]],\n i2: Tupler.Aux[Out0, Out],\n i3: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n val base = dataset.toDF()\n .select(columns.toList[UntypedExpression[T]].map(c => new Column(c.expr)):_*)\n val selected = base.as[Out](TypedExpressionEncoder[Out])\n\n TypedDataset.create[Out](selected)\n }\n }\n\n /** Sort each partition in the dataset using the columns selected. */\n def sortWithinPartitions[A: CatalystOrdered](ca: SortedTypedColumn[T, A]): TypedDataset[T] =\n sortWithinPartitionsMany(ca)\n\n /** Sort each partition in the dataset using the columns selected. */\n def sortWithinPartitions[A: CatalystOrdered, B: CatalystOrdered](\n ca: SortedTypedColumn[T, A],\n cb: SortedTypedColumn[T, B]\n ): TypedDataset[T] = sortWithinPartitionsMany(ca, cb)\n\n /** Sort each partition in the dataset using the columns selected. */\n def sortWithinPartitions[A: CatalystOrdered, B: CatalystOrdered, C: CatalystOrdered](\n ca: SortedTypedColumn[T, A],\n cb: SortedTypedColumn[T, B],\n cc: SortedTypedColumn[T, C]\n ): TypedDataset[T] = sortWithinPartitionsMany(ca, cb, cc)\n\n /** Sort each partition in the dataset by the given column expressions\n * Default sort order is ascending.\n * {{{\n * d.sortWithinPartitionsMany(d('a), d('b).desc, d('c).asc)\n * }}}\n */\n object sortWithinPartitionsMany extends ProductArgs {\n def applyProduct[U <: HList, O <: HList](columns: U)\n (implicit\n i0: Mapper.Aux[SortedTypedColumn.defaultAscendingPoly.type, U, O],\n i1: ToTraversable.Aux[O, List, SortedTypedColumn[T, _]]\n ): TypedDataset[T] = {\n val sorted = dataset.toDF()\n .sortWithinPartitions(i0(columns).toList[SortedTypedColumn[T, _]].map(_.untyped):_*)\n .as[T](TypedExpressionEncoder[T])\n\n TypedDataset.create[T](sorted)\n }\n }\n\n /** Orders the TypedDataset using the column selected. */\n def orderBy[A: CatalystOrdered](ca: SortedTypedColumn[T, A]): TypedDataset[T] =\n orderByMany(ca)\n\n /** Orders the TypedDataset using the columns selected. */\n def orderBy[A: CatalystOrdered, B: CatalystOrdered](\n ca: SortedTypedColumn[T, A],\n cb: SortedTypedColumn[T, B]\n ): TypedDataset[T] = orderByMany(ca, cb)\n\n /** Orders the TypedDataset using the columns selected. */\n def orderBy[A: CatalystOrdered, B: CatalystOrdered, C: CatalystOrdered](\n ca: SortedTypedColumn[T, A],\n cb: SortedTypedColumn[T, B],\n cc: SortedTypedColumn[T, C]\n ): TypedDataset[T] = orderByMany(ca, cb, cc)\n\n /** Sort the dataset by any number of column expressions.\n * Default sort order is ascending.\n * {{{\n * d.orderByMany(d('a), d('b).desc, d('c).asc)\n * }}}\n */\n object orderByMany extends ProductArgs {\n def applyProduct[U <: HList, O <: HList](columns: U)\n (implicit\n i0: Mapper.Aux[SortedTypedColumn.defaultAscendingPoly.type, U, O],\n i1: ToTraversable.Aux[O, List, SortedTypedColumn[T, _]]\n ): TypedDataset[T] = {\n val sorted = dataset.toDF()\n .orderBy(i0(columns).toList[SortedTypedColumn[T, _]].map(_.untyped):_*)\n .as[T](TypedExpressionEncoder[T])\n\n TypedDataset.create[T](sorted)\n }\n }\n\n /** Returns a new Dataset as a tuple with the specified\n * column dropped.\n * Does not allow for dropping from a single column TypedDataset\n *\n * {{{\n * val d: TypedDataset[Foo(a: String, b: Int...)] = ???\n * val result = TypedDataset[(Int, ...)] = d.drop('a)\n * }}}\n * @param column column to drop specified as a Symbol\n * @param i0 LabelledGeneric derived for T\n * @param i1 Remover derived for TRep and column\n * @param i2 values of T with column removed\n * @param i3 tupler of values\n * @param i4 evidence of encoder of the tupled values\n * @tparam Out Tupled return type\n * @tparam TRep shapeless' record representation of T\n * @tparam Removed record of T with column removed\n * @tparam ValuesFromRemoved values of T with column removed as an HList\n * @tparam V value type of column in T\n * @return\n */\n def dropTupled[Out, TRep <: HList, Removed <: HList, ValuesFromRemoved <: HList, V]\n (column: Witness.Lt[Symbol])\n (implicit\n i0: LabelledGeneric.Aux[T, TRep],\n i1: Remover.Aux[TRep, column.T, (V, Removed)],\n i2: Values.Aux[Removed, ValuesFromRemoved],\n i3: Tupler.Aux[ValuesFromRemoved, Out],\n i4: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n val dropped = dataset\n .toDF()\n .drop(column.value.name)\n .as[Out](TypedExpressionEncoder[Out])\n\n TypedDataset.create[Out](dropped)\n }\n\n /**\n * Drops columns as necessary to return `U`\n *\n * @example\n * {{{\n * case class X(i: Int, j: Int, k: Boolean)\n * case class Y(i: Int, k: Boolean)\n * val f: TypedDataset[X] = ???\n * val fNew: TypedDataset[Y] = f.drop[Y]\n * }}}\n *\n * @tparam U the output type\n *\n * @see [[frameless.TypedDataset#project]]\n */\n def drop[U](implicit projector: SmartProject[T,U]): TypedDataset[U] = project[U]\n\n /** Prepends a new column to the Dataset.\n *\n * {{{\n * case class X(i: Int, j: Int)\n * val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n * val fNew: TypedDataset[(Int,Int,Boolean)] = f.withColumnTupled(f('j) === 10)\n * }}}\n */\n def withColumnTupled[A: TypedEncoder, H <: HList, FH <: HList, Out]\n (ca: TypedColumn[T, A])\n (implicit\n i0: Generic.Aux[T, H],\n i1: Prepend.Aux[H, A :: HNil, FH],\n i2: Tupler.Aux[FH, Out],\n i3: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n // Giving a random name to the new column (the proper name will be given by the Tuple-based encoder)\n val selected = dataset.toDF().withColumn(\"I1X3T9CU1OP0128JYIO76TYZZA3AXHQ18RMI\", ca.untyped)\n .as[Out](TypedExpressionEncoder[Out])\n\n TypedDataset.create[Out](selected)\n }\n\n /** Returns a new [[frameless.TypedDataset]] with the specified column updated with a new value\n * {{{\n * case class X(i: Int, j: Int)\n * val f: TypedDataset[X] = TypedDataset.create(X(1,10) :: Nil)\n * val fNew: TypedDataset[X] = f.withColumn('j, f('i)) // results in X(1, 1) :: Nil\n * }}}\n * @param column column given as a symbol to replace\n * @param replacement column to replace the value with\n * @param i0 Evidence that a column with the correct type and name exists\n */\n def withColumnReplaced[A](\n column: Witness.Lt[Symbol],\n replacement: TypedColumn[T, A]\n )(implicit\n i0: TypedColumn.Exists[T, column.T, A]\n ): TypedDataset[T] = {\n val updated = dataset.toDF().withColumn(column.value.name, replacement.untyped)\n .as[T](TypedExpressionEncoder[T])\n\n TypedDataset.create[T](updated)\n }\n\n /** Adds a column to a Dataset so long as the specified output type, `U`, has\n * an extra column from `T` that has type `A`.\n *\n * @example\n * {{{\n * case class X(i: Int, j: Int)\n * case class Y(i: Int, j: Int, k: Boolean)\n * val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n * val fNew: TypedDataset[Y] = f.withColumn[Y](f('j) === 10)\n * }}}\n * @param ca The typed column to add\n * @param i0 TypeEncoder for output type U\n * @param i1 TypeEncoder for added column type A\n * @param i2 the LabelledGeneric derived for T\n * @param i3 the LabelledGeneric derived for U\n * @param i4 proof no fields have been removed\n * @param i5 diff from T to U\n * @param i6 keys from newFields\n * @param i7 the one and only new key\n * @param i8 the one and only new field enforcing the type of A exists\n * @param i9 the keys of U\n * @param iA allows for traversing the keys of U\n * @tparam U the output type\n * @tparam A The added column type\n * @tparam TRep shapeless' record representation of T\n * @tparam URep shapeless' record representation of U\n * @tparam UKeys the keys of U as an HList\n * @tparam NewFields the added fields to T to get U\n * @tparam NewKeys the keys of NewFields as an HList\n * @tparam NewKey the first, and only, key in NewKey\n *\n * @see [[frameless.TypedDataset.WithColumnApply#apply]]\n */\n def withColumn[U] = new WithColumnApply[U]\n\n class WithColumnApply[U] {\n def apply[A, TRep <: HList, URep <: HList, UKeys <: HList, NewFields <: HList, NewKeys <: HList, NewKey <: Symbol]\n (ca: TypedColumn[T, A])\n (implicit\n i0: TypedEncoder[U],\n i1: TypedEncoder[A],\n i2: LabelledGeneric.Aux[T, TRep],\n i3: LabelledGeneric.Aux[U, URep],\n i4: Diff.Aux[TRep, URep, HNil],\n i5: Diff.Aux[URep, TRep, NewFields],\n i6: Keys.Aux[NewFields, NewKeys],\n i7: IsHCons.Aux[NewKeys, NewKey, HNil],\n i8: IsHCons.Aux[NewFields, FieldType[NewKey, A], HNil],\n i9: Keys.Aux[URep, UKeys],\n iA: ToTraversable.Aux[UKeys, Seq, Symbol]\n ): TypedDataset[U] = {\n val newColumnName =\n i7.head(i6()).name\n\n val dfWithNewColumn = dataset\n .toDF()\n .withColumn(newColumnName, ca.untyped)\n\n val newColumns = i9.apply().to[Seq].map(_.name).map(dfWithNewColumn.col)\n\n val selected = dfWithNewColumn\n .select(newColumns: _*)\n .as[U](TypedExpressionEncoder[U])\n\n TypedDataset.create[U](selected)\n }\n }\n\n /**\n * Explodes a single column at a time. It only compiles if the type of column supports this operation.\n *\n * @example\n *\n * {{{\n * case class X(i: Int, j: Array[Int])\n * case class Y(i: Int, j: Int)\n *\n * val f: TypedDataset[X] = ???\n * val fNew: TypedDataset[Y] = f.explode('j).as[Y]\n * }}}\n * @param column the column we wish to explode\n */\n def explode[A, TRep <: HList, V[_], OutMod <: HList, OutModValues <: HList, Out]\n (column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, V[A]],\n i1: TypedEncoder[A],\n i2: CatalystExplodableCollection[V],\n i3: LabelledGeneric.Aux[T, TRep],\n i4: Modifier.Aux[TRep, column.T, V[A], A, OutMod],\n i5: Values.Aux[OutMod, OutModValues],\n i6: Tupler.Aux[OutModValues, Out],\n i7: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n import org.apache.spark.sql.functions.{explode => sparkExplode}\n val df = dataset.toDF()\n\n val trans =\n df\n .withColumn(column.value.name, sparkExplode(df(column.value.name)))\n .as[Out](TypedExpressionEncoder[Out])\n TypedDataset.create[Out](trans)\n }\n\n /**\n * Explodes a single column at a time. It only compiles if the type of column supports this operation.\n *\n * @example\n *\n * {{{\n * case class X(i: Int, j: Map[Int, Int])\n * case class Y(i: Int, j: (Int, Int))\n *\n * val f: TypedDataset[X] = ???\n * val fNew: TypedDataset[Y] = f.explodeMap('j).as[Y]\n * }}}\n * @param column the column we wish to explode\n */\n def explodeMap[A, B, V[_, _], TRep <: HList, OutMod <: HList, OutModValues <: HList, Out]\n (column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, V[A, B]],\n i1: TypedEncoder[A],\n i2: TypedEncoder[B],\n i3: LabelledGeneric.Aux[T, TRep],\n i4: Modifier.Aux[TRep, column.T, V[A,B], (A, B), OutMod],\n i5: Values.Aux[OutMod, OutModValues],\n i6: Tupler.Aux[OutModValues, Out],\n i7: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n import org.apache.spark.sql.functions.{explode => sparkExplode, struct => sparkStruct, col => sparkCol}\n val df = dataset.toDF()\n\n // select all columns, all original columns and [key, value] columns appeared after the map explode\n // .withColumn(column.value.name, sparkExplode(df(column.value.name))) in this case would not work\n // since the map explode produces two columns\n val columnNames = df.columns.toSeq\n val columnNamesRenamed = columnNames.map(c => s\"frameless_$c\")\n\n // preserve the original list of renamed columns\n val columns = columnNamesRenamed.map(sparkCol)\n\n val columnRenamed = s\"frameless_${column.value.name}\"\n // explode of a map adds \"key\" and \"value\" columns into the Row\n // this may cause col namings collision: row could already contain key / value columns\n // we rename the original Row columns to avoid this collision\n val dfr = df.toDF(columnNamesRenamed: _*)\n val exploded = dfr.select(sparkCol(\"*\"), sparkExplode(dfr(columnRenamed)))\n val trans =\n exploded\n // map explode explodes it into [key, value] columns\n // the only way to put it into a column is to create a struct\n .withColumn(columnRenamed, sparkStruct(exploded(\"key\"), exploded(\"value\")))\n // selecting only original columns, we don't need [key, value] columns left in the DataFrame after the map explode\n .select(columns: _*)\n // rename columns back and form the result\n .toDF(columnNames: _*)\n .as[Out](TypedExpressionEncoder[Out])\n TypedDataset.create[Out](trans)\n }\n\n /**\n * Flattens a column of type Option[A]. Compiles only if the selected column is of type Option[A].\n *\n *\n * @example\n *\n * {{{\n * case class X(i: Int, j: Option[Int])\n * case class Y(i: Int, j: Int)\n *\n * val f: TypedDataset[X] = ???\n * val fNew: TypedDataset[Y] = f.flattenOption('j).as[Y]\n * }}}\n *\n * @param column the column we wish to flatten\n */\n def flattenOption[A, TRep <: HList, V[_], OutMod <: HList, OutModValues <: HList, Out]\n (column: Witness.Lt[Symbol])\n (implicit\n i0: TypedColumn.Exists[T, column.T, V[A]],\n i1: TypedEncoder[A],\n i2: V[A] =:= Option[A],\n i3: LabelledGeneric.Aux[T, TRep],\n i4: Modifier.Aux[TRep, column.T, V[A], A, OutMod],\n i5: Values.Aux[OutMod, OutModValues],\n i6: Tupler.Aux[OutModValues, Out],\n i7: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n val df = dataset.toDF()\n val trans = df.filter(df(column.value.name).isNotNull).\n as[Out](TypedExpressionEncoder[Out])\n\n TypedDataset.create[Out](trans)\n }\n}\n\nobject TypedDataset {\n def create[A](data: Seq[A])\n (implicit\n encoder: TypedEncoder[A],\n sqlContext: SparkSession\n ): TypedDataset[A] = {\n val dataset = sqlContext.createDataset(data)(TypedExpressionEncoder[A])\n\n TypedDataset.create[A](dataset)\n }\n\n def create[A](data: RDD[A])\n (implicit\n encoder: TypedEncoder[A],\n sqlContext: SparkSession\n ): TypedDataset[A] = {\n val dataset = sqlContext.createDataset(data)(TypedExpressionEncoder[A])\n\n TypedDataset.create[A](dataset)\n }\n\n def create[A: TypedEncoder](dataset: Dataset[A]): TypedDataset[A] =\n createUnsafe(dataset.toDF())\n\n /**\n * Creates a [[frameless.TypedDataset]] from a Spark [[org.apache.spark.sql.DataFrame]].\n * Note that the names and types need to align!\n *\n * This is an unsafe operation: If the schemas do not align,\n * the error will be captured at runtime (not during compilation).\n */\n def createUnsafe[A: TypedEncoder](df: DataFrame): TypedDataset[A] = {\n val e = TypedEncoder[A]\n val output: Seq[Attribute] = df.queryExecution.analyzed.output\n\n val targetFields = TypedExpressionEncoder.targetStructType(e)\n val targetColNames: Seq[String] = targetFields.map(_.name)\n\n if (output.size != targetFields.size) {\n throw new IllegalStateException(\n s\"Unsupported creation of TypedDataset with ${targetFields.size} column(s) \" +\n s\"from a DataFrame with ${output.size} columns. \" +\n \"Try to `select()` the proper columns in the right order before calling `create()`.\")\n }\n\n // Adapt names if they are not the same (note: types still might not match)\n val shouldReshape = output.zip(targetColNames).exists {\n case (expr, colName) => expr.name != colName\n }\n val canSelect = targetColNames.toSet.subsetOf(output.map(_.name).toSet)\n\n val reshaped = if (shouldReshape && canSelect) {\n df.select(targetColNames.head, targetColNames.tail:_*)\n } else if (shouldReshape) {\n df.toDF(targetColNames: _*)\n } else {\n df\n }\n\n new TypedDataset[A](reshaped.as[A](TypedExpressionEncoder[A]))\n }\n\n /** Prefer `TypedDataset.create` over `TypedDataset.unsafeCreate` unless you\n * know what you are doing. */\n @deprecated(\"Prefer TypedDataset.create over TypedDataset.unsafeCreate\", \"0.3.0\")\n def unsafeCreate[A: TypedEncoder](dataset: Dataset[A]): TypedDataset[A] = {\n new TypedDataset[A](dataset)\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedDatasetForwarded.scala", "content": "package frameless\n\nimport java.util\n\nimport org.apache.spark.rdd.RDD\nimport org.apache.spark.sql.execution.QueryExecution\nimport org.apache.spark.sql.streaming.DataStreamWriter\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.sql.{DataFrame, DataFrameWriter, SQLContext, SparkSession}\nimport org.apache.spark.storage.StorageLevel\n\nimport scala.util.Random\n\n/** This trait implements [[TypedDataset]] methods that have the same signature\n * than their `Dataset` equivalent. Each method simply forwards the call to the\n * underlying `Dataset`.\n *\n * Documentation marked \"apache/spark\" is thanks to apache/spark Contributors\n * at https://github.com/apache/spark, licensed under Apache v2.0 available at\n * http://www.apache.org/licenses/LICENSE-2.0\n */\ntrait TypedDatasetForwarded[T] { self: TypedDataset[T] =>\n\n override def toString: String =\n dataset.toString\n\n /**\n * Returns a `SparkSession` from this [[TypedDataset]].\n */\n def sparkSession: SparkSession =\n dataset.sparkSession\n\n /**\n * Returns a `SQLContext` from this [[TypedDataset]].\n */\n def sqlContext: SQLContext =\n dataset.sqlContext\n\n /**\n * Returns the schema of this Dataset.\n *\n * apache/spark\n */\n def schema: StructType =\n dataset.schema\n\n /** Prints the schema of the underlying `Dataset` to the console in a nice tree format.\n *\n * apache/spark\n */\n def printSchema(): Unit =\n dataset.printSchema()\n\n /** Prints the plans (logical and physical) to the console for debugging purposes.\n *\n * apache/spark\n */\n def explain(extended: Boolean = false): Unit =\n dataset.explain(extended)\n\n /**\n * Returns a `QueryExecution` from this [[TypedDataset]].\n *\n * It is the primary workflow for executing relational queries using Spark. Designed to allow easy\n * access to the intermediate phases of query execution for developers.\n *\n * apache/spark\n */\n def queryExecution: QueryExecution =\n dataset.queryExecution\n\n /** Converts this strongly typed collection of data to generic Dataframe. In contrast to the\n * strongly typed objects that Dataset operations work on, a Dataframe returns generic Row\n * objects that allow fields to be accessed by ordinal or name.\n *\n * apache/spark\n */\n def toDF(): DataFrame =\n dataset.toDF()\n\n /** Converts this [[TypedDataset]] to an RDD.\n *\n * apache/spark\n */\n def rdd: RDD[T] =\n dataset.rdd\n\n /** Returns a new [[TypedDataset]] that has exactly `numPartitions` partitions.\n *\n * apache/spark\n */\n def repartition(numPartitions: Int): TypedDataset[T] =\n TypedDataset.create(dataset.repartition(numPartitions))\n\n\n /**\n * Get the [[TypedDataset]]'s current storage level, or StorageLevel.NONE if not persisted.\n *\n * apache/spark\n */\n def storageLevel(): StorageLevel =\n dataset.storageLevel\n\n /**\n * Returns the content of the [[TypedDataset]] as a Dataset of JSON strings.\n *\n * apache/spark\n */\n def toJSON: TypedDataset[String] =\n TypedDataset.create(dataset.toJSON)\n\n /**\n * Interface for saving the content of the non-streaming [[TypedDataset]] out into external storage.\n *\n * apache/spark\n */\n def write: DataFrameWriter[T] =\n dataset.write\n\n /**\n * Interface for saving the content of the streaming Dataset out into external storage.\n *\n * apache/spark\n */\n def writeStream: DataStreamWriter[T] =\n dataset.writeStream\n \n /** Returns a new [[TypedDataset]] that has exactly `numPartitions` partitions.\n * Similar to coalesce defined on an RDD, this operation results in a narrow dependency, e.g.\n * if you go from 1000 partitions to 100 partitions, there will not be a shuffle, instead each of\n * the 100 new partitions will claim 10 of the current partitions.\n *\n * apache/spark\n */\n def coalesce(numPartitions: Int): TypedDataset[T] =\n TypedDataset.create(dataset.coalesce(numPartitions))\n\n /**\n * Returns an `Array` that contains all column names in this [[TypedDataset]].\n */\n def columns: Array[String] =\n dataset.columns\n\n /** Concise syntax for chaining custom transformations.\n *\n * apache/spark\n */\n def transform[U](t: TypedDataset[T] => TypedDataset[U]): TypedDataset[U] =\n t(this)\n\n /** Returns a new Dataset by taking the first `n` rows. The difference between this function\n * and `head` is that `head` is an action and returns an array (by triggering query execution)\n * while `limit` returns a new Dataset.\n *\n * apache/spark\n */\n def limit(n: Int): TypedDataset[T] =\n TypedDataset.create(dataset.limit(n))\n\n /** Returns a new [[TypedDataset]] by sampling a fraction of records.\n *\n * apache/spark\n */\n def sample(withReplacement: Boolean, fraction: Double, seed: Long = Random.nextLong()): TypedDataset[T] =\n TypedDataset.create(dataset.sample(withReplacement, fraction, seed))\n\n /** Returns a new [[TypedDataset]] that contains only the unique elements of this [[TypedDataset]].\n *\n * Note that, equality checking is performed directly on the encoded representation of the data\n * and thus is not affected by a custom `equals` function defined on `T`.\n *\n * apache/spark\n */\n def distinct: TypedDataset[T] =\n TypedDataset.create(dataset.distinct())\n\n /**\n * Returns a best-effort snapshot of the files that compose this [[TypedDataset]]. This method simply\n * asks each constituent BaseRelation for its respective files and takes the union of all results.\n * Depending on the source relations, this may not find all input files. Duplicates are removed.\n *\n * apache/spark\n */\n\n def inputFiles: Array[String] =\n dataset.inputFiles\n\n /**\n * Returns true if the `collect` and `take` methods can be run locally\n * (without any Spark executors).\n *\n * apache/spark\n */\n def isLocal: Boolean =\n dataset.isLocal\n\n /**\n * Returns true if this [[TypedDataset]] contains one or more sources that continuously\n * return data as it arrives. A [[TypedDataset]] that reads data from a streaming source\n * must be executed as a `StreamingQuery` using the `start()` method in\n * `DataStreamWriter`. Methods that return a single answer, e.g. `count()` or\n * `collect()`, will throw an `AnalysisException` when there is a streaming\n * source present.\n *\n * apache/spark\n */\n def isStreaming: Boolean =\n dataset.isStreaming\n\n /** Returns a new [[TypedDataset]] that contains only the elements of this [[TypedDataset]] that are also\n * present in `other`.\n *\n * Note that, equality checking is performed directly on the encoded representation of the data\n * and thus is not affected by a custom `equals` function defined on `T`.\n *\n * apache/spark\n */\n def intersect(other: TypedDataset[T]): TypedDataset[T] =\n TypedDataset.create(dataset.intersect(other.dataset))\n\n /**\n * Randomly splits this [[TypedDataset]] with the provided weights.\n * Weights for splits, will be normalized if they don't sum to 1.\n *\n * apache/spark\n */\n // $COVERAGE-OFF$ We can not test this method because it is non-deterministic.\n def randomSplit(weights: Array[Double]): Array[TypedDataset[T]] =\n dataset.randomSplit(weights).map(TypedDataset.create[T])\n // $COVERAGE-ON$\n\n /**\n * Randomly splits this [[TypedDataset]] with the provided weights.\n * Weights for splits, will be normalized if they don't sum to 1.\n *\n * apache/spark\n */\n def randomSplit(weights: Array[Double], seed: Long): Array[TypedDataset[T]] =\n dataset.randomSplit(weights, seed).map(TypedDataset.create[T])\n\n /**\n * Returns a Java list that contains randomly split [[TypedDataset]] with the provided weights.\n * Weights for splits, will be normalized if they don't sum to 1.\n *\n * apache/spark\n */\n def randomSplitAsList(weights: Array[Double], seed: Long): util.List[TypedDataset[T]] = {\n val values = randomSplit(weights, seed)\n java.util.Arrays.asList(values: _*)\n }\n\n\n /** Returns a new Dataset containing rows in this Dataset but not in another Dataset.\n * This is equivalent to `EXCEPT` in SQL.\n *\n * Note that, equality checking is performed directly on the encoded representation of the data\n * and thus is not affected by a custom `equals` function defined on `T`.\n *\n * apache/spark\n */\n def except(other: TypedDataset[T]): TypedDataset[T] =\n TypedDataset.create(dataset.except(other.dataset))\n\n /** Persist this [[TypedDataset]] with the default storage level (`MEMORY_AND_DISK`).\n *\n * apache/spark\n */\n def cache(): TypedDataset[T] =\n TypedDataset.create(dataset.cache())\n\n /** Persist this [[TypedDataset]] with the given storage level.\n * @param newLevel One of: `MEMORY_ONLY`, `MEMORY_AND_DISK`, `MEMORY_ONLY_SER`,\n * `MEMORY_AND_DISK_SER`, `DISK_ONLY`, `MEMORY_ONLY_2`, `MEMORY_AND_DISK_2`, etc.\n *\n * apache/spark\n */\n def persist(newLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK): TypedDataset[T] =\n TypedDataset.create(dataset.persist(newLevel))\n\n /** Mark the [[TypedDataset]] as non-persistent, and remove all blocks for it from memory and disk.\n * @param blocking Whether to block until all blocks are deleted.\n *\n * apache/spark\n */\n def unpersist(blocking: Boolean = false): TypedDataset[T] =\n TypedDataset.create(dataset.unpersist(blocking))\n\n // $COVERAGE-OFF$ We do not test deprecated method since forwarded methods are tested.\n @deprecated(\"deserialized methods have moved to a separate section to highlight their runtime overhead\", \"0.4.0\")\n def map[U: TypedEncoder](func: T => U): TypedDataset[U] =\n deserialized.map(func)\n\n @deprecated(\"deserialized methods have moved to a separate section to highlight their runtime overhead\", \"0.4.0\")\n def mapPartitions[U: TypedEncoder](func: Iterator[T] => Iterator[U]): TypedDataset[U] =\n deserialized.mapPartitions(func)\n\n @deprecated(\"deserialized methods have moved to a separate section to highlight their runtime overhead\", \"0.4.0\")\n def flatMap[U: TypedEncoder](func: T => TraversableOnce[U]): TypedDataset[U] =\n deserialized.flatMap(func)\n\n @deprecated(\"deserialized methods have moved to a separate section to highlight their runtime overhead\", \"0.4.0\")\n def filter(func: T => Boolean): TypedDataset[T] =\n deserialized.filter(func)\n\n @deprecated(\"deserialized methods have moved to a separate section to highlight their runtime overhead\", \"0.4.0\")\n def reduceOption[F[_]: SparkDelay](func: (T, T) => T): F[Option[T]] =\n deserialized.reduceOption(func)\n // $COVERAGE-ON$\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n *\n * @example The correct way to do a projection on a single column is to\n * use the `select` method as follows:\n *\n * {{{\n * ds: TypedDataset[(String, String, String)] -> ds.select(ds('_2)).run()\n * }}}\n *\n * Spark provides an alternative way to obtain the same resulting `Dataset`,\n * using the `map` method:\n *\n * {{{\n * ds: TypedDataset[(String, String, String)] -> ds.deserialized.map(_._2).run()\n * }}}\n *\n * This second approach is however substantially slower than the first one,\n * and should be avoided as possible. Indeed, under the hood this `map` will\n * deserialize the entire `Tuple3` to an full JVM object, call the apply\n * method of the `_._2` closure on it, and serialize the resulting String back\n * to its Catalyst representation.\n */\n object deserialized {\n /** Returns a new [[TypedDataset]] that contains the result of applying `func` to each element.\n *\n * apache/spark\n */\n def map[U: TypedEncoder](func: T => U): TypedDataset[U] =\n TypedDataset.create(self.dataset.map(func)(TypedExpressionEncoder[U]))\n\n /** Returns a new [[TypedDataset]] that contains the result of applying `func` to each partition.\n *\n * apache/spark\n */\n def mapPartitions[U: TypedEncoder](func: Iterator[T] => Iterator[U]): TypedDataset[U] =\n TypedDataset.create(self.dataset.mapPartitions(func)(TypedExpressionEncoder[U]))\n\n /** Returns a new [[TypedDataset]] by first applying a function to all elements of this [[TypedDataset]],\n * and then flattening the results.\n *\n * apache/spark\n */\n def flatMap[U: TypedEncoder](func: T => TraversableOnce[U]): TypedDataset[U] =\n TypedDataset.create(self.dataset.flatMap(func)(TypedExpressionEncoder[U]))\n\n /** Returns a new [[TypedDataset]] that only contains elements where `func` returns `true`.\n *\n * apache/spark\n */\n def filter(func: T => Boolean): TypedDataset[T] =\n TypedDataset.create(self.dataset.filter(func))\n\n /** Optionally reduces the elements of this [[TypedDataset]] using the specified binary function. The given\n * `func` must be commutative and associative or the result may be non-deterministic.\n *\n * Differs from `Dataset#reduce` by wrapping its result into an `Option` and an effect-suspending `F`.\n */\n def reduceOption[F[_]](func: (T, T) => T)(implicit F: SparkDelay[F]): F[Option[T]] =\n F.delay {\n try {\n Option(self.dataset.reduce(func))\n } catch {\n case _: UnsupportedOperationException => None\n }\n }(self.dataset.sparkSession)\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedEncoder.scala", "content": "package frameless\n\nimport java.math.BigInteger\n\nimport java.util.Date\n\nimport java.time.{ Duration, Instant, Period, LocalDate }\n\nimport java.sql.Timestamp\n\nimport scala.reflect.ClassTag\n\nimport org.apache.spark.sql.FramelessInternals\nimport org.apache.spark.sql.FramelessInternals.UserDefinedType\nimport org.apache.spark.sql.{ reflection => ScalaReflection }\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.catalyst.expressions.objects._\nimport org.apache.spark.sql.catalyst.util.{\n ArrayBasedMapData,\n DateTimeUtils,\n GenericArrayData\n}\nimport org.apache.spark.sql.types._\nimport org.apache.spark.unsafe.types.UTF8String\n\nimport shapeless._\nimport shapeless.ops.hlist.IsHCons\n\nabstract class TypedEncoder[T](\n implicit\n val classTag: ClassTag[T])\n extends Serializable {\n def nullable: Boolean\n\n def jvmRepr: DataType\n def catalystRepr: DataType\n\n /**\n * From Catalyst representation to T\n */\n def fromCatalyst(path: Expression): Expression\n\n /**\n * T to Catalyst representation\n */\n def toCatalyst(path: Expression): Expression\n}\n\n// Waiting on scala 2.12\n// @annotation.implicitAmbiguous(msg =\n// \"\"\"TypedEncoder[${T}] can be obtained from automatic type class derivation, using the implicit Injection[${T}, ?] or using the implicit UserDefinedType[${T}] in scope.\n// To desambigious this resolution you need to either:\n// - Remove the implicit Injection[${T}, ?] from scope\n// - Remove the implicit UserDefinedType[${T}] from scope\n// - import TypedEncoder.usingInjection\n// - import TypedEncoder.usingDerivation\n// - import TypedEncoder.usingUserDefinedType\n// \"\"\")\nobject TypedEncoder {\n def apply[T: TypedEncoder]: TypedEncoder[T] = implicitly[TypedEncoder[T]]\n\n implicit val stringEncoder: TypedEncoder[String] = new TypedEncoder[String] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = FramelessInternals.objectTypeFor[String]\n def catalystRepr: DataType = StringType\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(classOf[UTF8String], catalystRepr, \"fromString\", path :: Nil)\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(path, \"toString\", jvmRepr)\n\n override val toString = \"stringEncoder\"\n }\n\n implicit val booleanEncoder: TypedEncoder[Boolean] =\n new TypedEncoder[Boolean] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = BooleanType\n def catalystRepr: DataType = BooleanType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val intEncoder: TypedEncoder[Int] = new TypedEncoder[Int] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = IntegerType\n def catalystRepr: DataType = IntegerType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n\n override def toString = \"intEncoder\"\n }\n\n implicit val longEncoder: TypedEncoder[Long] = new TypedEncoder[Long] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = LongType\n def catalystRepr: DataType = LongType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val shortEncoder: TypedEncoder[Short] = new TypedEncoder[Short] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ShortType\n def catalystRepr: DataType = ShortType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val charEncoder: TypedEncoder[Char] = new TypedEncoder[Char] {\n\n // tricky because while Char is primitive type, Spark doesn't support it\n implicit val charAsString: Injection[java.lang.Character, String] =\n new Injection[java.lang.Character, String] {\n def apply(a: java.lang.Character): String = String.valueOf(a)\n\n def invert(b: String): java.lang.Character = {\n require(b.length == 1)\n b.charAt(0)\n }\n }\n\n val underlying = usingInjection[java.lang.Character, String]\n\n def nullable: Boolean = false\n\n // this line fixes underlying encoder\n def jvmRepr: DataType =\n FramelessInternals.objectTypeFor[java.lang.Character]\n\n def catalystRepr: DataType = StringType\n\n def toCatalyst(path: Expression): Expression = underlying.toCatalyst(path)\n\n def fromCatalyst(path: Expression): Expression =\n underlying.fromCatalyst(path)\n }\n\n implicit val byteEncoder: TypedEncoder[Byte] = new TypedEncoder[Byte] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ByteType\n def catalystRepr: DataType = ByteType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val floatEncoder: TypedEncoder[Float] = new TypedEncoder[Float] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = FloatType\n def catalystRepr: DataType = FloatType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val doubleEncoder: TypedEncoder[Double] = new TypedEncoder[Double] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = DoubleType\n def catalystRepr: DataType = DoubleType\n\n def toCatalyst(path: Expression): Expression = path\n def fromCatalyst(path: Expression): Expression = path\n }\n\n implicit val bigDecimalEncoder: TypedEncoder[BigDecimal] =\n new TypedEncoder[BigDecimal] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[BigDecimal]\n def catalystRepr: DataType = DecimalType.SYSTEM_DEFAULT\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n Decimal.getClass,\n DecimalType.SYSTEM_DEFAULT,\n \"apply\",\n path :: Nil\n )\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(path, \"toBigDecimal\", jvmRepr)\n\n override def toString: String = \"bigDecimalEncoder\"\n }\n\n implicit val javaBigDecimalEncoder: TypedEncoder[java.math.BigDecimal] =\n new TypedEncoder[java.math.BigDecimal] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[java.math.BigDecimal]\n def catalystRepr: DataType = DecimalType.SYSTEM_DEFAULT\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n Decimal.getClass,\n DecimalType.SYSTEM_DEFAULT,\n \"apply\",\n path :: Nil\n )\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(path, \"toJavaBigDecimal\", jvmRepr)\n\n override def toString: String = \"javaBigDecimalEncoder\"\n }\n\n implicit val bigIntEncoder: TypedEncoder[BigInt] = new TypedEncoder[BigInt] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[BigInt]\n def catalystRepr: DataType = DecimalType(DecimalType.MAX_PRECISION, 0)\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n Decimal.getClass,\n catalystRepr,\n \"apply\",\n path :: Nil\n )\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(path, \"toScalaBigInt\", jvmRepr)\n\n override def toString: String = \"bigIntEncoder\"\n }\n\n implicit val javaBigIntEncoder: TypedEncoder[BigInteger] =\n new TypedEncoder[BigInteger] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[BigInteger]\n def catalystRepr: DataType = DecimalType(DecimalType.MAX_PRECISION, 0)\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n Decimal.getClass,\n catalystRepr,\n \"apply\",\n path :: Nil\n )\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(path, \"toJavaBigInteger\", jvmRepr)\n\n override def toString: String = \"javaBigIntEncoder\"\n }\n\n implicit val sqlDate: TypedEncoder[SQLDate] = new TypedEncoder[SQLDate] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[SQLDate]\n def catalystRepr: DataType = DateType\n\n def toCatalyst(path: Expression): Expression =\n Invoke(path, \"days\", DateType)\n\n def fromCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = SQLDate.getClass,\n dataType = jvmRepr,\n functionName = \"apply\",\n arguments = path :: Nil,\n propagateNull = true\n )\n }\n\n implicit val timestampEncoder: TypedEncoder[Timestamp] =\n new TypedEncoder[Timestamp] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[Timestamp]\n def catalystRepr: DataType = TimestampType\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n DateTimeUtils.getClass,\n TimestampType,\n \"fromJavaTimestamp\",\n path :: Nil,\n returnNullable = false\n )\n\n def fromCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = DateTimeUtils.getClass,\n dataType = jvmRepr,\n functionName = \"toJavaTimestamp\",\n arguments = path :: Nil,\n propagateNull = true\n )\n\n override def toString: String = \"timestampEncoder\"\n }\n\n implicit val dateEncoder: TypedEncoder[Date] = new TypedEncoder[Date] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[Date]\n def catalystRepr: DataType = TimestampType\n\n private val instantRepr = ScalaReflection.dataTypeFor[Instant]\n\n def toCatalyst(path: Expression): Expression =\n timeInstant.toCatalyst(Invoke(path, \"toInstant\", instantRepr))\n\n def fromCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = classOf[Date],\n dataType = jvmRepr,\n functionName = \"from\",\n arguments = timeInstant.fromCatalyst(path) :: Nil,\n propagateNull = true\n )\n\n override def toString: String = \"dateEncoder\"\n }\n\n implicit val sqlDateEncoder: TypedEncoder[java.sql.Date] =\n new TypedEncoder[java.sql.Date] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[java.sql.Date]\n def catalystRepr: DataType = DateType\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = DateTimeUtils.getClass,\n dataType = catalystRepr,\n functionName = \"fromJavaDate\",\n arguments = path :: Nil,\n propagateNull = true\n )\n\n private val localDateRepr = ScalaReflection.dataTypeFor[LocalDate]\n\n def fromCatalyst(path: Expression): Expression = {\n val toLocalDate = StaticInvoke(\n staticObject = DateTimeUtils.getClass,\n dataType = localDateRepr,\n functionName = \"daysToLocalDate\",\n arguments = path :: Nil,\n propagateNull = true\n )\n\n StaticInvoke(\n staticObject = classOf[java.sql.Date],\n dataType = jvmRepr,\n functionName = \"valueOf\",\n arguments = toLocalDate :: Nil,\n propagateNull = true\n )\n }\n\n override def toString: String = \"sqlDateEncoder\"\n }\n\n implicit val sqlTimestamp: TypedEncoder[SQLTimestamp] =\n new TypedEncoder[SQLTimestamp] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[SQLTimestamp]\n def catalystRepr: DataType = TimestampType\n\n def toCatalyst(path: Expression): Expression =\n Invoke(path, \"us\", TimestampType)\n\n def fromCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = SQLTimestamp.getClass,\n dataType = jvmRepr,\n functionName = \"apply\",\n arguments = path :: Nil,\n propagateNull = true\n )\n }\n\n /** java.time Encoders, Spark uses https://github.com/apache/spark/blob/v3.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala for encoding / decoding. */\n implicit val timeInstant: TypedEncoder[Instant] = new TypedEncoder[Instant] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = ScalaReflection.dataTypeFor[Instant]\n def catalystRepr: DataType = TimestampType\n\n def toCatalyst(path: Expression): Expression =\n StaticInvoke(\n DateTimeUtils.getClass,\n TimestampType,\n \"instantToMicros\",\n path :: Nil,\n returnNullable = false\n )\n\n def fromCatalyst(path: Expression): Expression =\n StaticInvoke(\n staticObject = DateTimeUtils.getClass,\n dataType = jvmRepr,\n functionName = \"microsToInstant\",\n arguments = path :: Nil,\n propagateNull = true\n )\n }\n\n /**\n * DayTimeIntervalType and YearMonthIntervalType in Spark 3.2.0.\n * We maintain Spark 3.x cross compilation and handle Duration and Period as an injections to be compatible with Spark versions < 3.2\n * See\n * * https://github.com/apache/spark/blob/v3.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala#L1031-L1047\n * * https://github.com/apache/spark/blob/v3.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala#L1075-L1087\n */\n // DayTimeIntervalType\n implicit val timeDurationInjection: Injection[Duration, Long] =\n Injection(_.toMillis, Duration.ofMillis)\n\n // YearMonthIntervalType\n implicit val timePeriodInjection: Injection[Period, Int] =\n Injection(_.getDays, Period.ofDays)\n\n implicit val timePeriodEncoder: TypedEncoder[Period] =\n TypedEncoder.usingInjection\n\n implicit val timeDurationEncoder: TypedEncoder[Duration] =\n TypedEncoder.usingInjection\n\n implicit def arrayEncoder[T: ClassTag](\n implicit\n i0: Lazy[RecordFieldEncoder[T]]\n ): TypedEncoder[Array[T]] =\n new TypedEncoder[Array[T]] {\n private lazy val encodeT = i0.value.encoder\n\n def nullable: Boolean = false\n\n lazy val jvmRepr: DataType = i0.value.jvmRepr match {\n case ByteType => BinaryType\n case _ => FramelessInternals.objectTypeFor[Array[T]]\n }\n\n lazy val catalystRepr: DataType = i0.value.jvmRepr match {\n case ByteType => BinaryType\n case _ => ArrayType(encodeT.catalystRepr, encodeT.nullable)\n }\n\n def toCatalyst(path: Expression): Expression = {\n val enc = i0.value\n\n enc.jvmRepr match {\n case IntegerType | LongType | DoubleType | FloatType | ShortType |\n BooleanType =>\n StaticInvoke(\n classOf[UnsafeArrayData],\n catalystRepr,\n \"fromPrimitiveArray\",\n path :: Nil\n )\n\n case ByteType => path\n\n case _ =>\n MapObjects(enc.toCatalyst, path, enc.jvmRepr, encodeT.nullable)\n }\n }\n\n def fromCatalyst(path: Expression): Expression =\n encodeT.jvmRepr match {\n case IntegerType => Invoke(path, \"toIntArray\", jvmRepr)\n case LongType => Invoke(path, \"toLongArray\", jvmRepr)\n case DoubleType => Invoke(path, \"toDoubleArray\", jvmRepr)\n case FloatType => Invoke(path, \"toFloatArray\", jvmRepr)\n case ShortType => Invoke(path, \"toShortArray\", jvmRepr)\n case BooleanType => Invoke(path, \"toBooleanArray\", jvmRepr)\n\n case ByteType => path\n\n case _ =>\n Invoke(\n MapObjects(\n i0.value.fromCatalyst,\n path,\n encodeT.catalystRepr,\n encodeT.nullable\n ),\n \"array\",\n jvmRepr\n )\n }\n\n override def toString: String = s\"arrayEncoder($jvmRepr)\"\n }\n\n implicit def collectionEncoder[C[X] <: Seq[X], T](\n implicit\n i0: Lazy[RecordFieldEncoder[T]],\n i1: ClassTag[C[T]]\n ): TypedEncoder[C[T]] = new TypedEncoder[C[T]] {\n private lazy val encodeT = i0.value.encoder\n\n def nullable: Boolean = false\n\n def jvmRepr: DataType = FramelessInternals.objectTypeFor[C[T]](i1)\n\n def catalystRepr: DataType =\n ArrayType(encodeT.catalystRepr, encodeT.nullable)\n\n def toCatalyst(path: Expression): Expression = {\n val enc = i0.value\n\n if (ScalaReflection.isNativeType(enc.jvmRepr)) {\n NewInstance(classOf[GenericArrayData], path :: Nil, catalystRepr)\n } else {\n MapObjects(enc.toCatalyst, path, enc.jvmRepr, encodeT.nullable)\n }\n }\n\n def fromCatalyst(path: Expression): Expression =\n MapObjects(\n i0.value.fromCatalyst,\n path,\n encodeT.catalystRepr,\n encodeT.nullable,\n Some(i1.runtimeClass) // This will cause MapObjects to build a collection of type C[_] directly\n )\n\n override def toString: String = s\"collectionEncoder($jvmRepr)\"\n }\n\n /**\n * @param i1 implicit lazy `RecordFieldEncoder[T]` to encode individual elements of the set.\n * @param i2 implicit `ClassTag[Set[T]]` to provide runtime information about the set type.\n * @tparam T the element type of the set.\n * @return a `TypedEncoder` instance for `Set[T]`.\n */\n implicit def setEncoder[T](\n implicit\n i1: shapeless.Lazy[RecordFieldEncoder[T]],\n i2: ClassTag[Set[T]]\n ): TypedEncoder[Set[T]] = {\n implicit val inj: Injection[Set[T], Seq[T]] = Injection(_.toSeq, _.toSet)\n\n TypedEncoder.usingInjection\n }\n\n /**\n * @tparam A the key type\n * @tparam B the value type\n * @param i0 the keys encoder\n * @param i1 the values encoder\n */\n implicit def mapEncoder[A: NotCatalystNullable, B](\n implicit\n i0: Lazy[RecordFieldEncoder[A]],\n i1: Lazy[RecordFieldEncoder[B]]\n ): TypedEncoder[Map[A, B]] = new TypedEncoder[Map[A, B]] {\n def nullable: Boolean = false\n\n def jvmRepr: DataType = FramelessInternals.objectTypeFor[Map[A, B]]\n\n private lazy val encodeA = i0.value.encoder\n private lazy val encodeB = i1.value.encoder\n\n lazy val catalystRepr: DataType =\n MapType(encodeA.catalystRepr, encodeB.catalystRepr, encodeB.nullable)\n\n def fromCatalyst(path: Expression): Expression = {\n val keyArrayType = ArrayType(encodeA.catalystRepr, containsNull = false)\n\n val keyData = Invoke(\n MapObjects(\n i0.value.fromCatalyst,\n Invoke(path, \"keyArray\", keyArrayType),\n encodeA.catalystRepr\n ),\n \"array\",\n FramelessInternals.objectTypeFor[Array[Any]]\n )\n\n val valueArrayType = ArrayType(encodeB.catalystRepr, encodeB.nullable)\n\n val valueData = Invoke(\n MapObjects(\n i1.value.fromCatalyst,\n Invoke(path, \"valueArray\", valueArrayType),\n encodeB.catalystRepr\n ),\n \"array\",\n FramelessInternals.objectTypeFor[Array[Any]]\n )\n\n StaticInvoke(\n ArrayBasedMapData.getClass,\n jvmRepr,\n \"toScalaMap\",\n keyData :: valueData :: Nil\n )\n }\n\n def toCatalyst(path: Expression): Expression = {\n val encA = i0.value\n val encB = i1.value\n\n ExternalMapToCatalyst(\n path,\n encA.jvmRepr,\n encA.toCatalyst,\n false,\n encB.jvmRepr,\n encB.toCatalyst,\n encodeB.nullable\n )\n }\n\n override def toString = s\"mapEncoder($jvmRepr)\"\n }\n\n implicit def optionEncoder[A](\n implicit\n underlying: TypedEncoder[A]\n ): TypedEncoder[Option[A]] =\n new TypedEncoder[Option[A]] {\n def nullable: Boolean = true\n\n def jvmRepr: DataType =\n FramelessInternals.objectTypeFor[Option[A]](classTag)\n\n def catalystRepr: DataType = underlying.catalystRepr\n\n def toCatalyst(path: Expression): Expression = {\n // for primitive types we must manually unbox the value of the object\n underlying.jvmRepr match {\n case IntegerType =>\n Invoke(\n UnwrapOption(\n ScalaReflection.dataTypeFor[java.lang.Integer],\n path\n ),\n \"intValue\",\n IntegerType\n )\n\n case LongType =>\n Invoke(\n UnwrapOption(ScalaReflection.dataTypeFor[java.lang.Long], path),\n \"longValue\",\n LongType\n )\n\n case DoubleType =>\n Invoke(\n UnwrapOption(ScalaReflection.dataTypeFor[java.lang.Double], path),\n \"doubleValue\",\n DoubleType\n )\n\n case FloatType =>\n Invoke(\n UnwrapOption(ScalaReflection.dataTypeFor[java.lang.Float], path),\n \"floatValue\",\n FloatType\n )\n\n case ShortType =>\n Invoke(\n UnwrapOption(ScalaReflection.dataTypeFor[java.lang.Short], path),\n \"shortValue\",\n ShortType\n )\n\n case ByteType =>\n Invoke(\n UnwrapOption(ScalaReflection.dataTypeFor[java.lang.Byte], path),\n \"byteValue\",\n ByteType\n )\n\n case BooleanType =>\n Invoke(\n UnwrapOption(\n ScalaReflection.dataTypeFor[java.lang.Boolean],\n path\n ),\n \"booleanValue\",\n BooleanType\n )\n\n case _ =>\n underlying.toCatalyst(UnwrapOption(underlying.jvmRepr, path))\n }\n }\n\n def fromCatalyst(path: Expression): Expression =\n WrapOption(underlying.fromCatalyst(path), underlying.jvmRepr)\n }\n\n /** Encodes things using injection if there is one defined */\n implicit def usingInjection[A: ClassTag, B](\n implicit\n inj: Injection[A, B],\n trb: TypedEncoder[B]\n ): TypedEncoder[A] =\n new TypedEncoder[A] {\n def nullable: Boolean = trb.nullable\n def jvmRepr: DataType = FramelessInternals.objectTypeFor[A](classTag)\n def catalystRepr: DataType = trb.catalystRepr\n\n def fromCatalyst(path: Expression): Expression = {\n val bexpr = trb.fromCatalyst(path)\n Invoke(Literal.fromObject(inj), \"invert\", jvmRepr, Seq(bexpr))\n }\n\n def toCatalyst(path: Expression): Expression =\n trb.toCatalyst(\n Invoke(Literal.fromObject(inj), \"apply\", trb.jvmRepr, Seq(path))\n )\n }\n\n /** Encodes things as records if there is no Injection defined */\n implicit def usingDerivation[F, G <: HList, H <: HList](\n implicit\n i0: LabelledGeneric.Aux[F, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons[H],\n i3: Lazy[RecordEncoderFields[H]],\n i4: Lazy[NewInstanceExprs[G]],\n i5: ClassTag[F]\n ): TypedEncoder[F] = new RecordEncoder[F, G, H]\n\n /** Encodes things using a Spark SQL's User Defined Type (UDT) if there is one defined in implicit */\n implicit def usingUserDefinedType[\n A >: Null: UserDefinedType: ClassTag\n ]: TypedEncoder[A] = {\n val udt = implicitly[UserDefinedType[A]]\n val udtInstance =\n NewInstance(udt.getClass, Nil, dataType = ObjectType(udt.getClass))\n\n new TypedEncoder[A] {\n def nullable: Boolean = false\n def jvmRepr: DataType = ObjectType(udt.userClass)\n def catalystRepr: DataType = udt\n\n def toCatalyst(path: Expression): Expression =\n Invoke(udtInstance, \"serialize\", udt, Seq(path))\n\n def fromCatalyst(path: Expression): Expression =\n Invoke(udtInstance, \"deserialize\", ObjectType(udt.userClass), Seq(path))\n }\n }\n\n object injections extends InjectionEnum\n}\n" }, { "path": "dataset/src/main/scala/frameless/TypedExpressionEncoder.scala", "content": "package frameless\n\nimport org.apache.spark.sql.Encoder\nimport org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal\nimport org.apache.spark.sql.catalyst.encoders.ExpressionEncoder\nimport org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStruct, If}\nimport org.apache.spark.sql.types.StructType\n\nobject TypedExpressionEncoder {\n\n /** In Spark, DataFrame has always schema of StructType\n *\n * DataFrames of primitive types become records \n * with a single field called \"value\" set in ExpressionEncoder.\n */\n def targetStructType[A](encoder: TypedEncoder[A]): StructType =\n encoder.catalystRepr match {\n case x: StructType =>\n if (encoder.nullable) StructType(x.fields.map(_.copy(nullable = true)))\n else x\n\n case dt => new StructType().add(\"value\", dt, nullable = encoder.nullable)\n }\n\n def apply[T](implicit encoder: TypedEncoder[T]): Encoder[T] = {\n val in = BoundReference(0, encoder.jvmRepr, encoder.nullable)\n\n val (out, serializer) = encoder.toCatalyst(in) match {\n case it @ If(_, _, _: CreateNamedStruct) => {\n val out = GetColumnByOrdinal(0, encoder.catalystRepr)\n\n out -> it\n }\n\n case other => {\n val out = GetColumnByOrdinal(0, encoder.catalystRepr)\n\n out -> other\n }\n }\n\n new ExpressionEncoder[T](\n objSerializer = serializer,\n objDeserializer = encoder.fromCatalyst(out),\n clsTag = encoder.classTag\n )\n }\n}\n\n" }, { "path": "dataset/src/main/scala/frameless/With.scala", "content": "package frameless\n\n/** Compute the intersection of two types:\n *\n * - With[A, A] = A\n * - With[A, B] = A with B (when A != B)\n *\n * This type function is needed to prevent IDEs from infering large types\n * with shape `A with A with ... with A`. These types could be confusing for\n * both end users and IDE's type checkers.\n */\ntrait With[A, B] { type Out }\n\nobject With extends LowPrioWith {\n implicit def combine[A, B]: Aux[A, B, A with B] = of[A, B, A with B]\n}\n\nprivate[frameless] sealed trait LowPrioWith {\n type Aux[A, B, W] = With[A, B] { type Out = W }\n\n protected[this] val theInstance = new With[Any, Any] {}\n\n protected[this] def of[A, B, W]: With[A, B] { type Out = W } =\n theInstance.asInstanceOf[Aux[A, B, W]]\n\n implicit def identity[T]: Aux[T, T, T] = of[T, T, T]\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/AggregateFunctions.scala", "content": "package frameless\npackage functions\n\nimport org.apache.spark.sql.FramelessInternals.expr\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.{functions => sparkFunctions}\nimport frameless.syntax._\n\nimport scala.annotation.nowarn\n\ntrait AggregateFunctions {\n /** Aggregate function: returns the number of items in a group.\n *\n * apache/spark\n */\n def count[T](): TypedAggregate[T, Long] =\n sparkFunctions.count(sparkFunctions.lit(1)).typedAggregate\n\n /** Aggregate function: returns the number of items in a group for which the selected column is not null.\n *\n * apache/spark\n */\n def count[T](column: TypedColumn[T, _]): TypedAggregate[T, Long] =\n sparkFunctions.count(column.untyped).typedAggregate\n\n /** Aggregate function: returns the number of distinct items in a group.\n *\n * apache/spark\n */\n def countDistinct[T](column: TypedColumn[T, _]): TypedAggregate[T, Long] =\n sparkFunctions.countDistinct(column.untyped).typedAggregate\n\n /** Aggregate function: returns the approximate number of distinct items in a group.\n */\n def approxCountDistinct[T](column: TypedColumn[T, _]): TypedAggregate[T, Long] =\n sparkFunctions.approx_count_distinct(column.untyped).typedAggregate\n\n /** Aggregate function: returns the approximate number of distinct items in a group.\n *\n * @param rsd maximum estimation error allowed (default = 0.05)\n *\n * apache/spark\n */\n def approxCountDistinct[T](column: TypedColumn[T, _], rsd: Double): TypedAggregate[T, Long] =\n sparkFunctions.approx_count_distinct(column.untyped, rsd).typedAggregate\n\n /** Aggregate function: returns a list of objects with duplicates.\n *\n * apache/spark\n */\n def collectList[T, A: TypedEncoder](column: TypedColumn[T, A]): TypedAggregate[T, Vector[A]] =\n sparkFunctions.collect_list(column.untyped).typedAggregate\n\n /** Aggregate function: returns a set of objects with duplicate elements eliminated.\n *\n * apache/spark\n */\n def collectSet[T, A: TypedEncoder](column: TypedColumn[T, A]): TypedAggregate[T, Vector[A]] =\n sparkFunctions.collect_set(column.untyped).typedAggregate\n\n /** Aggregate function: returns the sum of all values in the given column.\n *\n * apache/spark\n */\n def sum[A, T, Out](column: TypedColumn[T, A])(\n implicit\n summable: CatalystSummable[A, Out],\n oencoder: TypedEncoder[Out],\n aencoder: TypedEncoder[A]\n ): TypedAggregate[T, Out] = {\n val zeroExpr = Literal.create(summable.zero, TypedEncoder[A].catalystRepr)\n val sumExpr = expr(sparkFunctions.sum(column.untyped))\n val sumOrZero = Coalesce(Seq(sumExpr, zeroExpr))\n\n new TypedAggregate[T, Out](sumOrZero)\n }\n\n /** Aggregate function: returns the sum of distinct values in the column.\n *\n * apache/spark\n */\n @nowarn // supress sparkFunctions.sumDistinct call which is used to maintain Spark 3.1.x backwards compat\n def sumDistinct[A, T, Out](column: TypedColumn[T, A])(\n implicit\n summable: CatalystSummable[A, Out],\n oencoder: TypedEncoder[Out],\n aencoder: TypedEncoder[A]\n ): TypedAggregate[T, Out] = {\n val zeroExpr = Literal.create(summable.zero, TypedEncoder[A].catalystRepr)\n val sumExpr = expr(sparkFunctions.sumDistinct(column.untyped))\n val sumOrZero = Coalesce(Seq(sumExpr, zeroExpr))\n\n new TypedAggregate[T, Out](sumOrZero)\n }\n\n /** Aggregate function: returns the average of the values in a group.\n *\n * apache/spark\n */\n def avg[A, T, Out](column: TypedColumn[T, A])(\n implicit\n averageable: CatalystAverageable[A, Out],\n oencoder: TypedEncoder[Out]\n ): TypedAggregate[T, Out] = {\n new TypedAggregate[T, Out](sparkFunctions.avg(column.untyped))\n }\n\n /** Aggregate function: returns the unbiased variance of the values in a group.\n *\n * @note In Spark variance always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#186]]\n *\n * apache/spark\n */\n def variance[A: CatalystVariance, T](column: TypedColumn[T, A]): TypedAggregate[T, Double] =\n sparkFunctions.variance(column.untyped).typedAggregate\n\n /** Aggregate function: returns the sample standard deviation.\n *\n * @note In Spark stddev always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#155]]\n *\n * apache/spark\n */\n def stddev[A: CatalystVariance, T](column: TypedColumn[T, A]): TypedAggregate[T, Double] =\n sparkFunctions.stddev(column.untyped).typedAggregate\n\n /**\n * Aggregate function: returns the standard deviation of a column by population.\n *\n * @note In Spark stddev always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#L143]]\n *\n * apache/spark\n */\n def stddevPop[A, T](column: TypedColumn[T, A])(implicit ev: CatalystCast[A, Double]): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.stddev_pop(column.cast[Double].untyped)\n )\n }\n\n /**\n * Aggregate function: returns the standard deviation of a column by sample.\n *\n * @note In Spark stddev always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#L160]]\n *\n * apache/spark\n */\n def stddevSamp[A, T](column: TypedColumn[T, A])(implicit ev: CatalystCast[A, Double] ): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.stddev_samp(column.cast[Double].untyped)\n )\n }\n\n /** Aggregate function: returns the maximum value of the column in a group.\n *\n * apache/spark\n */\n def max[A: CatalystOrdered, T](column: TypedColumn[T, A]): TypedAggregate[T, A] = {\n implicit val c = column.uencoder\n sparkFunctions.max(column.untyped).typedAggregate\n }\n\n /** Aggregate function: returns the minimum value of the column in a group.\n *\n * apache/spark\n */\n def min[A: CatalystOrdered, T](column: TypedColumn[T, A]): TypedAggregate[T, A] = {\n implicit val c = column.uencoder\n sparkFunctions.min(column.untyped).typedAggregate\n }\n\n /** Aggregate function: returns the first value in a group.\n *\n * The function by default returns the first values it sees. It will return the first non-null\n * value it sees when ignoreNulls is set to true. If all values are null, then null is returned.\n *\n * apache/spark\n */\n def first[A, T](column: TypedColumn[T, A]): TypedAggregate[T, A] = {\n sparkFunctions.first(column.untyped).typedAggregate(column.uencoder)\n }\n\n /**\n * Aggregate function: returns the last value in a group.\n *\n * The function by default returns the last values it sees. It will return the last non-null\n * value it sees when ignoreNulls is set to true. If all values are null, then null is returned.\n *\n * apache/spark\n */\n def last[A, T](column: TypedColumn[T, A]): TypedAggregate[T, A] = {\n implicit val c = column.uencoder\n sparkFunctions.last(column.untyped).typedAggregate\n }\n\n /**\n * Aggregate function: returns the Pearson Correlation Coefficient for two columns.\n *\n * @note In Spark corr always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala#L95]]\n *\n * apache/spark\n */\n def corr[A, B, T](column1: TypedColumn[T, A], column2: TypedColumn[T, B])\n (implicit\n i0: CatalystCast[A, Double],\n i1: CatalystCast[B, Double]\n ): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.corr(column1.cast[Double].untyped, column2.cast[Double].untyped)\n )\n }\n\n /**\n * Aggregate function: returns the covariance of two collumns.\n *\n * @note In Spark covar_pop always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala#L82]]\n *\n * apache/spark\n */\n def covarPop[A, B, T](column1: TypedColumn[T, A], column2: TypedColumn[T, B])\n (implicit\n i0: CatalystCast[A, Double],\n i1: CatalystCast[B, Double]\n ): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.covar_pop(column1.cast[Double].untyped, column2.cast[Double].untyped)\n )\n }\n\n /**\n * Aggregate function: returns the covariance of two columns.\n *\n * @note In Spark covar_samp always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala#L93]]\n *\n * apache/spark\n */\n def covarSamp[A, B, T](column1: TypedColumn[T, A], column2: TypedColumn[T, B])\n (implicit\n i0: CatalystCast[A, Double],\n i1: CatalystCast[B, Double]\n ): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.covar_samp(column1.cast[Double].untyped, column2.cast[Double].untyped)\n )\n }\n\n\n /**\n * Aggregate function: returns the kurtosis of a column.\n *\n * @note In Spark kurtosis always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#L220]]\n *\n * apache/spark\n */\n def kurtosis[A, T](column: TypedColumn[T, A])(implicit ev: CatalystCast[A, Double]): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.kurtosis(column.cast[Double].untyped)\n )\n }\n\n /**\n * Aggregate function: returns the skewness of a column.\n *\n * @note In Spark skewness always returns Double\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala#L200]]\n *\n * apache/spark\n */\n def skewness[A, T](column: TypedColumn[T, A])(implicit ev: CatalystCast[A, Double]): TypedAggregate[T, Option[Double]] = {\n new TypedAggregate[T, Option[Double]](\n sparkFunctions.skewness(column.cast[Double].untyped)\n )\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/Lit.scala", "content": "package frameless.functions\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.catalyst.expressions.codegen._\nimport org.apache.spark.sql.catalyst.expressions.{Expression, NonSQLExpression}\nimport org.apache.spark.sql.types.DataType\n\nprivate[frameless] case class Lit[T <: AnyVal](\n dataType: DataType,\n nullable: Boolean,\n show: () => String,\n catalystExpr: Expression // must be a generated Expression from a literal TypedEncoder's toCatalyst function\n) extends Expression with NonSQLExpression {\n override def toString: String = s\"FramelessLit(${show()})\"\n\n lazy val codegen = {\n val ctx = new CodegenContext()\n val eval = genCode(ctx)\n\n val codeBody =\n s\"\"\"\n public scala.Function1 generate(Object[] references) {\n return new LiteralEvalImpl(references);\n }\n\n class LiteralEvalImpl extends scala.runtime.AbstractFunction1 {\n private final Object[] references;\n ${ctx.declareMutableStates()}\n ${ctx.declareAddedFunctions()}\n\n public LiteralEvalImpl(Object[] references) {\n this.references = references;\n ${ctx.initMutableStates()}\n }\n\n public java.lang.Object apply(java.lang.Object z) {\n InternalRow ${ctx.INPUT_ROW} = (InternalRow) z;\n ${eval.code}\n return ${eval.isNull} ? ((Object)null) : ((Object)${eval.value});\n }\n }\n \"\"\"\n\n val code = CodeFormatter.stripOverlappingComments(\n new CodeAndComment(codeBody, ctx.getPlaceHolderToComments())\n )\n\n val (clazz, _) = CodeGenerator.compile(code)\n val codegen =\n clazz.generate(ctx.references.toArray).asInstanceOf[InternalRow => AnyRef]\n codegen\n }\n\n def eval(input: InternalRow): Any = codegen(input)\n \n def children: Seq[Expression] = Nil\n\n protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = catalystExpr.genCode(ctx)\n\n protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = this\n\n override val foldable: Boolean = catalystExpr.foldable\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/NonAggregateFunctions.scala", "content": "package frameless\npackage functions\n\nimport org.apache.spark.sql.{Column, functions => sparkFunctions}\n\nimport scala.annotation.nowarn\nimport scala.util.matching.Regex\n\ntrait NonAggregateFunctions {\n /** Non-Aggregate function: calculates the SHA-2 digest of a binary column and returns the value as a 40 character hex string\n *\n * apache/spark\n */\n def sha2[T](column: AbstractTypedColumn[T, Array[Byte]], numBits: Int): column.ThisType[T, String] =\n column.typed(sparkFunctions.sha2(column.untyped, numBits))\n\n /** Non-Aggregate function: calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex string\n *\n * apache/spark\n */\n def sha1[T](column: AbstractTypedColumn[T, Array[Byte]]): column.ThisType[T, String] =\n column.typed(sparkFunctions.sha1(column.untyped))\n\n /** Non-Aggregate function: returns a cyclic redundancy check value of a binary column as long.\n *\n * apache/spark\n */\n def crc32[T](column: AbstractTypedColumn[T, Array[Byte]]): column.ThisType[T, Long] =\n column.typed(sparkFunctions.crc32(column.untyped))\n /**\n * Non-Aggregate function: returns the negated value of column.\n *\n * apache/spark\n */\n def negate[A, B, T](column: AbstractTypedColumn[T,A])(\n implicit i0: CatalystNumericWithJavaBigDecimal[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T,B] =\n column.typed(sparkFunctions.negate(column.untyped))\n\n /**\n * Non-Aggregate function: logical not.\n *\n * apache/spark\n */\n def not[T](column: AbstractTypedColumn[T,Boolean]): column.ThisType[T,Boolean] =\n column.typed(sparkFunctions.not(column.untyped))\n\n /**\n * Non-Aggregate function: Convert a number in a string column from one base to another.\n *\n * apache/spark\n */\n def conv[T](column: AbstractTypedColumn[T,String], fromBase: Int, toBase: Int): column.ThisType[T,String] =\n column.typed(sparkFunctions.conv(column.untyped,fromBase,toBase))\n\n /** Non-Aggregate function: Converts an angle measured in radians to an approximately equivalent angle measured in degrees.\n *\n * apache/spark\n */\n def degrees[A,T](column: AbstractTypedColumn[T,A]): column.ThisType[T,Double] =\n column.typed(sparkFunctions.degrees(column.untyped))\n\n /** Non-Aggregate function: returns the ceiling of a numeric column\n *\n * apache/spark\n */\n def ceil[A, B, T](column: AbstractTypedColumn[T, A])\n (implicit\n i0: CatalystRound[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.ceil(column.untyped))(i1)\n\n /** Non-Aggregate function: returns the floor of a numeric column\n *\n * apache/spark\n */\n def floor[A, B, T](column: AbstractTypedColumn[T, A])\n (implicit\n i0: CatalystRound[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.floor(column.untyped))(i1)\n\n /** Non-Aggregate function: unsigned shift the the given value numBits right. If given long, will return long else it will return an integer.\n *\n * apache/spark\n */\n @nowarn // supress sparkFunctions.shiftRightUnsigned call which is used to maintain Spark 3.1.x backwards compat\n def shiftRightUnsigned[A, B, T](column: AbstractTypedColumn[T, A], numBits: Int)\n (implicit\n i0: CatalystBitShift[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.shiftRightUnsigned(column.untyped, numBits))\n\n /** Non-Aggregate function: shift the the given value numBits right. If given long, will return long else it will return an integer.\n *\n * apache/spark\n */\n @nowarn // supress sparkFunctions.shiftReft call which is used to maintain Spark 3.1.x backwards compat\n def shiftRight[A, B, T](column: AbstractTypedColumn[T, A], numBits: Int)\n (implicit\n i0: CatalystBitShift[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.shiftRight(column.untyped, numBits))\n\n /** Non-Aggregate function: shift the the given value numBits left. If given long, will return long else it will return an integer.\n *\n * apache/spark\n */\n @nowarn // supress sparkFunctions.shiftLeft call which is used to maintain Spark 3.1.x backwards compat\n def shiftLeft[A, B, T](column: AbstractTypedColumn[T, A], numBits: Int)\n (implicit\n i0: CatalystBitShift[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.shiftLeft(column.untyped, numBits))\n \n /** Non-Aggregate function: returns the absolute value of a numeric column\n *\n * apache/spark\n */\n def abs[A, B, T](column: AbstractTypedColumn[T, A])\n (implicit\n i0: CatalystNumericWithJavaBigDecimal[A, B],\n i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.abs(column.untyped))(i1)\n\n /** Non-Aggregate function: Computes the cosine of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def cos[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.cos(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the hyperbolic cosine of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def cosh[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.cosh(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the signum of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def signum[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.signum(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the sine of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def sin[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.sin(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the hyperbolic sine of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def sinh[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.sinh(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the tangent of the given column.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def tan[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.tan(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Computes the hyperbolic tangent of the given value.\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def tanh[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.tanh(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the acos of a numeric column\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def acos[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.acos(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns true if value is contained with in the array in the specified column\n *\n * apache/spark\n */\n def arrayContains[C[_]: CatalystCollection, A, T](column: AbstractTypedColumn[T, C[A]], value: A): column.ThisType[T, Boolean] =\n column.typed(sparkFunctions.array_contains(column.untyped, value))\n\n /** Non-Aggregate function: returns the atan of a numeric column\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def atan[A, T](column: AbstractTypedColumn[T,A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.atan(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the asin of a numeric column\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def asin[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.asin(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the angle theta from the conversion of rectangular coordinates (x, y) to\n * polar coordinates (r, theta).\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def atan2[A, B, T](l: TypedColumn[T, A], r: TypedColumn[T, B])\n (implicit\n i0: CatalystCast[A, Double],\n i1: CatalystCast[B, Double]\n ): TypedColumn[T, Double] =\n r.typed(sparkFunctions.atan2(l.cast[Double].untyped, r.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the angle theta from the conversion of rectangular coordinates (x, y) to\n * polar coordinates (r, theta).\n *\n * Spark will expect a Double value for this expression. See:\n * [[https://github.com/apache/spark/blob/4a3c09601ba69f7d49d1946bb6f20f5cfe453031/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala#L67]]\n * apache/spark\n */\n def atan2[A, B, T](l: TypedAggregate[T, A], r: TypedAggregate[T, B])\n (implicit\n i0: CatalystCast[A, Double],\n i1: CatalystCast[B, Double]\n ): TypedAggregate[T, Double] =\n r.typed(sparkFunctions.atan2(l.cast[Double].untyped, r.cast[Double].untyped))\n\n def atan2[B, T](l: Double, r: TypedColumn[T, B])\n (implicit i0: CatalystCast[B, Double]): TypedColumn[T, Double] =\n atan2(r.lit(l), r)\n\n def atan2[A, T](l: TypedColumn[T, A], r: Double)\n (implicit i0: CatalystCast[A, Double]): TypedColumn[T, Double] =\n atan2(l, l.lit(r))\n\n def atan2[B, T](l: Double, r: TypedAggregate[T, B])\n (implicit i0: CatalystCast[B, Double]): TypedAggregate[T, Double] =\n atan2(r.lit(l), r)\n\n def atan2[A, T](l: TypedAggregate[T, A], r: Double)\n (implicit i0: CatalystCast[A, Double]): TypedAggregate[T, Double] =\n atan2(l, l.lit(r))\n\n /** Non-Aggregate function: returns the square root value of a numeric column.\n *\n * apache/spark\n */\n def sqrt[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.sqrt(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the cubic root value of a numeric column.\n *\n * apache/spark\n */\n def cbrt[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.cbrt(column.cast[Double].untyped))\n\n /** Non-Aggregate function: returns the exponential value of a numeric column.\n *\n * apache/spark\n */\n def exp[A, T](column: AbstractTypedColumn[T, A])\n (implicit i0: CatalystCast[A, Double]): column.ThisType[T, Double] =\n column.typed(sparkFunctions.exp(column.cast[Double].untyped))\n\n /** Non-Aggregate function: Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode.\n *\n * apache/spark\n */\n def round[A, B, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystNumericWithJavaBigDecimal[A, B], i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.round(column.untyped))(i1)\n\n /** Non-Aggregate function: Round the value of `e` to `scale` decimal places with HALF_UP round mode\n * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.\n *\n * apache/spark\n */\n def round[A, B, T](column: AbstractTypedColumn[T, A], scale: Int)(\n implicit i0: CatalystNumericWithJavaBigDecimal[A, B], i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.round(column.untyped, scale))(i1)\n\n /** Non-Aggregate function: Bankers Rounding - returns the rounded to 0 decimal places value with HALF_EVEN round mode\n * of a numeric column.\n *\n * apache/spark\n */\n def bround[A, B, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystNumericWithJavaBigDecimal[A, B], i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.bround(column.untyped))(i1)\n\n /** Non-Aggregate function: Bankers Rounding - returns the rounded to `scale` decimal places value with HALF_EVEN round mode\n * of a numeric column. If `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.\n *\n * apache/spark\n */\n def bround[A, B, T](column: AbstractTypedColumn[T, A], scale: Int)(\n implicit i0: CatalystNumericWithJavaBigDecimal[A, B], i1: TypedEncoder[B]\n ): column.ThisType[T, B] =\n column.typed(sparkFunctions.bround(column.untyped, scale))(i1)\n\n /**\n * Computes the natural logarithm of the given value.\n *\n * apache/spark\n */\n def log[A, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.log(column.untyped))\n\n /**\n * Returns the first argument-base logarithm of the second argument.\n *\n * apache/spark\n */\n def log[A, T](base: Double, column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.log(base, column.untyped))\n\n /**\n * Computes the logarithm of the given column in base 2.\n *\n * apache/spark\n */\n def log2[A, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.log2(column.untyped))\n\n /**\n * Computes the natural logarithm of the given value plus one.\n *\n * apache/spark\n */\n def log1p[A, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.log1p(column.untyped))\n\n /**\n * Computes the logarithm of the given column in base 10.\n *\n * apache/spark\n */\n def log10[A, T](column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.log10(column.untyped))\n\n\n /**\n * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.\n *\n * apache/spark\n */\n def hypot[A, T](column: AbstractTypedColumn[T, A], column2: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.hypot(column.untyped, column2.untyped))\n\n /**\n * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.\n *\n * apache/spark\n */\n def hypot[A, T](column: AbstractTypedColumn[T, A], l: Double)(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.hypot(column.untyped, l))\n\n /**\n * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.\n *\n * apache/spark\n */\n def hypot[A, T](l: Double, column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.hypot(l, column.untyped))\n\n /**\n * Returns the value of the first argument raised to the power of the second argument.\n *\n * apache/spark\n */\n def pow[A, T](column: AbstractTypedColumn[T, A], column2: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.pow(column.untyped, column2.untyped))\n\n /**\n * Returns the value of the first argument raised to the power of the second argument.\n *\n * apache/spark\n */\n def pow[A, T](column: AbstractTypedColumn[T, A], l: Double)(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.pow(column.untyped, l))\n\n /**\n * Returns the value of the first argument raised to the power of the second argument.\n *\n * apache/spark\n */\n def pow[A, T](l: Double, column: AbstractTypedColumn[T, A])(\n implicit i0: CatalystCast[A, Double]\n ): column.ThisType[T, Double] =\n column.typed(sparkFunctions.pow(l, column.untyped))\n\n /**\n * Returns the positive value of dividend mod divisor.\n *\n * apache/spark\n */\n def pmod[A, T](column: AbstractTypedColumn[T, A], column2: AbstractTypedColumn[T, A])(\n implicit i0: TypedEncoder[A]\n ): column.ThisType[T, A] =\n column.typed(sparkFunctions.pmod(column.untyped, column2.untyped))\n\n\n /** Non-Aggregate function: Returns the string representation of the binary value of the given long\n * column. For example, bin(\"12\") returns \"1100\".\n *\n * apache/spark\n */\n def bin[T](column: AbstractTypedColumn[T, Long]): column.ThisType[T, String] =\n column.typed(sparkFunctions.bin(column.untyped))\n\n /**\n * Calculates the MD5 digest of a binary column and returns the value\n * as a 32 character hex string.\n *\n * apache/spark\n */\n def md5[T, A](column: AbstractTypedColumn[T, A])(implicit i0: TypedEncoder[A]): column.ThisType[T, String] =\n column.typed(sparkFunctions.md5(column.untyped))\n\n /**\n * Computes the factorial of the given value.\n *\n * apache/spark\n */\n def factorial[T](column: AbstractTypedColumn[T, Long])(implicit i0: TypedEncoder[Long]): column.ThisType[T, Long] =\n column.typed(sparkFunctions.factorial(column.untyped))\n\n /** Non-Aggregate function: Computes bitwise NOT.\n *\n * apache/spark\n */\n @nowarn // supress sparkFunctions.bitwiseNOT call which is used to maintain Spark 3.1.x backwards compat\n def bitwiseNOT[A: CatalystBitwise, T](column: AbstractTypedColumn[T, A]): column.ThisType[T, A] =\n column.typed(sparkFunctions.bitwiseNOT(column.untyped))(column.uencoder)\n\n /** Non-Aggregate function: file name of the current Spark task. Empty string if row did not originate from\n * a file\n *\n * apache/spark\n */\n def inputFileName[T](): TypedColumn[T, String] =\n new TypedColumn[T, String](sparkFunctions.input_file_name())\n\n /** Non-Aggregate function: generates monotonically increasing id\n *\n * apache/spark\n */\n def monotonicallyIncreasingId[T](): TypedColumn[T, Long] = {\n new TypedColumn[T, Long](sparkFunctions.monotonically_increasing_id())\n }\n\n /** Non-Aggregate function: Evaluates a list of conditions and returns one of multiple\n * possible result expressions. If none match, otherwise is returned\n * {{{\n * when(ds('boolField), ds('a))\n * .when(ds('otherBoolField), lit(123))\n * .otherwise(ds('b))\n * }}}\n * apache/spark\n */\n def when[T, A](condition: AbstractTypedColumn[T, Boolean], value: AbstractTypedColumn[T, A]): When[T, A] =\n new When[T, A](condition, value)\n\n class When[T, A] private (untypedC: Column) {\n private[functions] def this(condition: AbstractTypedColumn[T, Boolean], value: AbstractTypedColumn[T, A]) =\n this(sparkFunctions.when(condition.untyped, value.untyped))\n\n def when(condition: AbstractTypedColumn[T, Boolean], value: AbstractTypedColumn[T, A]): When[T, A] =\n new When[T, A](untypedC.when(condition.untyped, value.untyped))\n\n def otherwise(value: AbstractTypedColumn[T, A]): value.ThisType[T, A] =\n value.typed(untypedC.otherwise(value.untyped))(value.uencoder)\n }\n\n //////////////////////////////////////////////////////////////////////////////////////////////\n // String functions\n //////////////////////////////////////////////////////////////////////////////////////////////\n\n\n /** Non-Aggregate function: takes the first letter of a string column and returns the ascii int value in a new column\n *\n * apache/spark\n */\n def ascii[T](column: AbstractTypedColumn[T, String]): column.ThisType[T, Int] =\n column.typed(sparkFunctions.ascii(column.untyped))\n\n /** Non-Aggregate function: Computes the BASE64 encoding of a binary column and returns it as a string column.\n * This is the reverse of unbase64.\n *\n * apache/spark\n */\n def base64[T](column: AbstractTypedColumn[T, Array[Byte]]): column.ThisType[T, String] =\n column.typed(sparkFunctions.base64(column.untyped))\n\n /** Non-Aggregate function: Decodes a BASE64 encoded string column and returns it as a binary column.\n * This is the reverse of base64.\n *\n * apache/spark\n */\n def unbase64[T](column: AbstractTypedColumn[T, String]): column.ThisType[T, Array[Byte]] =\n column.typed(sparkFunctions.unbase64(column.untyped))\n\n /** Non-Aggregate function: Concatenates multiple input string columns together into a single string column.\n * @note varargs make it harder to generalize so we overload the method for [[TypedColumn]] and [[TypedAggregate]]\n *\n * apache/spark\n */\n def concat[T](columns: TypedColumn[T, String]*): TypedColumn[T, String] =\n new TypedColumn(sparkFunctions.concat(columns.map(_.untyped): _*))\n\n /** Non-Aggregate function: Concatenates multiple input string columns together into a single string column.\n * @note varargs make it harder to generalize so we overload the method for [[TypedColumn]] and [[TypedAggregate]]\n *\n * apache/spark\n */\n def concat[T](columns: TypedAggregate[T, String]*): TypedAggregate[T, String] =\n new TypedAggregate(sparkFunctions.concat(columns.map(_.untyped): _*))\n\n /** Non-Aggregate function: Concatenates multiple input string columns together into a single string column,\n * using the given separator.\n * @note varargs make it harder to generalize so we overload the method for [[TypedColumn]] and [[TypedAggregate]]\n *\n * apache/spark\n */\n def concatWs[T](sep: String, columns: TypedAggregate[T, String]*): TypedAggregate[T, String] =\n new TypedAggregate(sparkFunctions.concat_ws(sep, columns.map(_.untyped): _*))\n\n /** Non-Aggregate function: Concatenates multiple input string columns together into a single string column,\n * using the given separator.\n * @note varargs make it harder to generalize so we overload the method for [[TypedColumn]] and [[TypedAggregate]]\n *\n * apache/spark\n */\n def concatWs[T](sep: String, columns: TypedColumn[T, String]*): TypedColumn[T, String] =\n new TypedColumn(sparkFunctions.concat_ws(sep, columns.map(_.untyped): _*))\n\n /** Non-Aggregate function: Locates the position of the first occurrence of substring column\n * in given string\n *\n * @note The position is not zero based, but 1 based index. Returns 0 if substr\n * could not be found in str.\n *\n * apache/spark\n */\n def instr[T](str: AbstractTypedColumn[T, String], substring: String): str.ThisType[T, Int] =\n str.typed(sparkFunctions.instr(str.untyped, substring))\n\n /** Non-Aggregate function: Computes the length of a given string.\n *\n * apache/spark\n */\n //TODO: Also for binary\n def length[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Int] =\n str.typed(sparkFunctions.length(str.untyped))\n\n /** Non-Aggregate function: Computes the Levenshtein distance of the two given string columns.\n *\n * apache/spark\n */\n def levenshtein[T](l: TypedColumn[T, String], r: TypedColumn[T, String]): TypedColumn[T, Int] =\n l.typed(sparkFunctions.levenshtein(l.untyped, r.untyped))\n\n /** Non-Aggregate function: Computes the Levenshtein distance of the two given string columns.\n *\n * apache/spark\n */\n def levenshtein[T](l: TypedAggregate[T, String], r: TypedAggregate[T, String]): TypedAggregate[T, Int] =\n l.typed(sparkFunctions.levenshtein(l.untyped, r.untyped))\n\n /** Non-Aggregate function: Converts a string column to lower case.\n *\n * apache/spark\n */\n def lower[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.lower(str.untyped))\n\n /** Non-Aggregate function: Left-pad the string column with pad to a length of len. If the string column is longer\n * than len, the return value is shortened to len characters.\n *\n * apache/spark\n */\n def lpad[T](str: AbstractTypedColumn[T, String],\n len: Int,\n pad: String): str.ThisType[T, String] =\n str.typed(sparkFunctions.lpad(str.untyped, len, pad))\n\n /** Non-Aggregate function: Trim the spaces from left end for the specified string value.\n *\n * apache/spark\n */\n def ltrim[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.ltrim(str.untyped))\n\n /** Non-Aggregate function: Replace all substrings of the specified string value that match regexp with rep.\n *\n * apache/spark\n */\n def regexpReplace[T](str: AbstractTypedColumn[T, String],\n pattern: Regex,\n replacement: String): str.ThisType[T, String] =\n str.typed(sparkFunctions.regexp_replace(str.untyped, pattern.regex, replacement))\n\n\n /** Non-Aggregate function: Reverses the string column and returns it as a new string column.\n *\n * apache/spark\n */\n def reverse[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.reverse(str.untyped))\n\n /** Non-Aggregate function: Right-pad the string column with pad to a length of len.\n * If the string column is longer than len, the return value is shortened to len characters.\n *\n * apache/spark\n */\n def rpad[T](str: AbstractTypedColumn[T, String], len: Int, pad: String): str.ThisType[T, String] =\n str.typed(sparkFunctions.rpad(str.untyped, len, pad))\n\n /** Non-Aggregate function: Trim the spaces from right end for the specified string value.\n *\n * apache/spark\n */\n def rtrim[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.rtrim(str.untyped))\n\n /** Non-Aggregate function: Substring starts at `pos` and is of length `len`\n *\n * apache/spark\n */\n //TODO: Also for byte array\n def substring[T](str: AbstractTypedColumn[T, String], pos: Int, len: Int): str.ThisType[T, String] =\n str.typed(sparkFunctions.substring(str.untyped, pos, len))\n\n /** Non-Aggregate function: Trim the spaces from both ends for the specified string column.\n *\n * apache/spark\n */\n def trim[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.trim(str.untyped))\n\n /** Non-Aggregate function: Converts a string column to upper case.\n *\n * apache/spark\n */\n def upper[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, String] =\n str.typed(sparkFunctions.upper(str.untyped))\n\n //////////////////////////////////////////////////////////////////////////////////////////////\n // DateTime functions\n //////////////////////////////////////////////////////////////////////////////////////////////\n\n /** Non-Aggregate function: Extracts the year as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#year` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def year[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.year(str.untyped))\n\n /** Non-Aggregate function: Extracts the quarter as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#quarter` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def quarter[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.quarter(str.untyped))\n\n /** Non-Aggregate function Extracts the month as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#month` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def month[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.month(str.untyped))\n\n /** Non-Aggregate function: Extracts the day of the week as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#dayofweek` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def dayofweek[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.dayofweek(str.untyped))\n\n /** Non-Aggregate function: Extracts the day of the month as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#dayofmonth` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def dayofmonth[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.dayofmonth(str.untyped))\n\n /** Non-Aggregate function: Extracts the day of the year as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#dayofyear` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def dayofyear[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.dayofyear(str.untyped))\n\n /** Non-Aggregate function: Extracts the hours as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#hour` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def hour[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.hour(str.untyped))\n\n /** Non-Aggregate function: Extracts the minutes as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#minute` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def minute[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.minute(str.untyped))\n\n /** Non-Aggregate function: Extracts the seconds as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#second` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def second[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.second(str.untyped))\n\n /** Non-Aggregate function: Extracts the week number as an integer from a given date/timestamp/string.\n *\n * Differs from `Column#weekofyear` by wrapping it's result into an `Option`.\n *\n * apache/spark\n */\n def weekofyear[T](str: AbstractTypedColumn[T, String]): str.ThisType[T, Option[Int]] =\n str.typed(sparkFunctions.weekofyear(str.untyped))\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/Udf.scala", "content": "package frameless\npackage functions\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.catalyst.expressions.{Expression, LeafExpression, NonSQLExpression}\nimport org.apache.spark.sql.catalyst.expressions.codegen._\nimport Block._\nimport org.apache.spark.sql.types.DataType\nimport shapeless.syntax.std.tuple._\n\n/** Documentation marked \"apache/spark\" is thanks to apache/spark Contributors\n * at https://github.com/apache/spark, licensed under Apache v2.0 available at\n * http://www.apache.org/licenses/LICENSE-2.0\n */\ntrait Udf {\n\n /** Defines a user-defined function of 1 arguments as user-defined function (UDF).\n * The data types are automatically inferred based on the function's signature.\n *\n * apache/spark\n */\n def udf[T, A, R: TypedEncoder](f: A => R):\n TypedColumn[T, A] => TypedColumn[T, R] = {\n u =>\n val scalaUdf = FramelessUdf(f, List(u), TypedEncoder[R])\n new TypedColumn[T, R](scalaUdf)\n }\n\n /** Defines a user-defined function of 2 arguments as user-defined function (UDF).\n * The data types are automatically inferred based on the function's signature.\n *\n * apache/spark\n */\n def udf[T, A1, A2, R: TypedEncoder](f: (A1,A2) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2]) => TypedColumn[T, R] = {\n case us =>\n val scalaUdf = FramelessUdf(f, us.toList[UntypedExpression[T]], TypedEncoder[R])\n new TypedColumn[T, R](scalaUdf)\n }\n\n /** Defines a user-defined function of 3 arguments as user-defined function (UDF).\n * The data types are automatically inferred based on the function's signature.\n *\n * apache/spark\n */\n def udf[T, A1, A2, A3, R: TypedEncoder](f: (A1,A2,A3) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3]) => TypedColumn[T, R] = {\n case us =>\n val scalaUdf = FramelessUdf(f, us.toList[UntypedExpression[T]], TypedEncoder[R])\n new TypedColumn[T, R](scalaUdf)\n }\n\n /** Defines a user-defined function of 4 arguments as user-defined function (UDF).\n * The data types are automatically inferred based on the function's signature.\n *\n * apache/spark\n */\n def udf[T, A1, A2, A3, A4, R: TypedEncoder](f: (A1,A2,A3,A4) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4]) => TypedColumn[T, R] = {\n case us =>\n val scalaUdf = FramelessUdf(f, us.toList[UntypedExpression[T]], TypedEncoder[R])\n new TypedColumn[T, R](scalaUdf)\n }\n\n /** Defines a user-defined function of 5 arguments as user-defined function (UDF).\n * The data types are automatically inferred based on the function's signature.\n *\n * apache/spark\n */\n def udf[T, A1, A2, A3, A4, A5, R: TypedEncoder](f: (A1,A2,A3,A4,A5) => R):\n (TypedColumn[T, A1], TypedColumn[T, A2], TypedColumn[T, A3], TypedColumn[T, A4], TypedColumn[T, A5]) => TypedColumn[T, R] = {\n case us =>\n val scalaUdf = FramelessUdf(f, us.toList[UntypedExpression[T]], TypedEncoder[R])\n new TypedColumn[T, R](scalaUdf)\n }\n}\n\n/**\n * NB: Implementation detail, isn't intended to be directly used.\n *\n * Our own implementation of `ScalaUDF` from Catalyst compatible with [[TypedEncoder]].\n */\ncase class FramelessUdf[T, R](\n function: AnyRef,\n encoders: Seq[TypedEncoder[_]],\n children: Seq[Expression],\n rencoder: TypedEncoder[R]\n) extends Expression with NonSQLExpression {\n\n override def nullable: Boolean = rencoder.nullable\n override def toString: String = s\"FramelessUdf(${children.mkString(\", \")})\"\n\n lazy val evalCode = {\n val ctx = new CodegenContext()\n val eval = genCode(ctx)\n\n val codeBody = s\"\"\"\n public scala.Function1 generate(Object[] references) {\n return new FramelessUdfEvalImpl(references);\n }\n\n class FramelessUdfEvalImpl extends scala.runtime.AbstractFunction1 {\n private final Object[] references;\n ${ctx.declareMutableStates()}\n ${ctx.declareAddedFunctions()}\n\n public FramelessUdfEvalImpl(Object[] references) {\n this.references = references;\n ${ctx.initMutableStates()}\n }\n\n public java.lang.Object apply(java.lang.Object z) {\n InternalRow ${ctx.INPUT_ROW} = (InternalRow) z;\n ${eval.code}\n return ${eval.isNull} ? ((Object)null) : ((Object)${eval.value});\n }\n }\n \"\"\"\n\n val code = CodeFormatter.stripOverlappingComments(\n new CodeAndComment(codeBody, ctx.getPlaceHolderToComments()))\n\n val (clazz, _) = CodeGenerator.compile(code)\n val codegen = clazz.generate(ctx.references.toArray).asInstanceOf[InternalRow => AnyRef]\n\n codegen\n }\n\n def eval(input: InternalRow): Any = {\n evalCode(input)\n }\n\n def dataType: DataType = rencoder.catalystRepr\n\n override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {\n ctx.references += this\n\n // save reference to `function` field from `FramelessUdf` to call it later\n val framelessUdfClassName = classOf[FramelessUdf[_, _]].getName\n val funcClassName = s\"scala.Function${children.size}\"\n val funcExpressionIdx = ctx.references.size - 1\n val funcTerm = ctx.addMutableState(funcClassName, ctx.freshName(\"udf\"),\n v => s\"$v = ($funcClassName)((($framelessUdfClassName)references\" +\n s\"[$funcExpressionIdx]).function());\")\n\n val (argsCode, funcArguments) = encoders.zip(children).map {\n case (encoder, child) =>\n val eval = child.genCode(ctx)\n val codeTpe = CodeGenerator.boxedType(encoder.jvmRepr)\n val argTerm = ctx.freshName(\"arg\")\n val convert = s\"${eval.code}\\n$codeTpe $argTerm = ${eval.isNull} ? (($codeTpe)null) : (($codeTpe)(${eval.value}));\"\n\n (convert, argTerm)\n }.unzip\n\n val internalTpe = CodeGenerator.boxedType(rencoder.jvmRepr)\n val internalTerm = ctx.addMutableState(internalTpe, ctx.freshName(\"internal\"))\n val internalNullTerm = ctx.addMutableState(\"boolean\", ctx.freshName(\"internalNull\"))\n // CTw - can't inject the term, may have to duplicate old code for parity\n val internalExpr = Spark2_4_LambdaVariable(internalTerm, internalNullTerm, rencoder.jvmRepr, true)\n\n val resultEval = rencoder.toCatalyst(internalExpr).genCode(ctx)\n\n ev.copy(code = code\"\"\"\n ${argsCode.mkString(\"\\n\")}\n\n $internalTerm =\n ($internalTpe)$funcTerm.apply(${funcArguments.mkString(\", \")});\n $internalNullTerm = $internalTerm == null;\n\n ${resultEval.code}\n \"\"\",\n value = resultEval.value,\n isNull = resultEval.isNull\n )\n }\n\n protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(children = newChildren)\n}\n\ncase class Spark2_4_LambdaVariable(\n value: String,\n isNull: String,\n dataType: DataType,\n nullable: Boolean = true) extends LeafExpression with NonSQLExpression {\n\n private val accessor: (InternalRow, Int) => Any = InternalRow.getAccessor(dataType)\n\n // Interpreted execution of `LambdaVariable` always get the 0-index element from input row.\n override def eval(input: InternalRow): Any = {\n assert(input.numFields == 1,\n \"The input row of interpreted LambdaVariable should have only 1 field.\")\n if (nullable && input.isNullAt(0)) {\n null\n } else {\n accessor(input, 0)\n }\n }\n\n override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {\n val isNullValue = if (nullable) {\n JavaCode.isNullVariable(isNull)\n } else {\n FalseLiteral\n }\n ExprCode(value = JavaCode.variable(value, dataType), isNull = isNullValue)\n }\n}\n\nobject FramelessUdf {\n // Spark needs case class with `children` field to mutate it\n def apply[T, R](\n function: AnyRef,\n cols: Seq[UntypedExpression[T]],\n rencoder: TypedEncoder[R]\n ): FramelessUdf[T, R] = FramelessUdf(\n function = function,\n encoders = cols.map(_.uencoder).toList,\n children = cols.map(x => x.uencoder.fromCatalyst(x.expr)).toList,\n rencoder = rencoder\n )\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/UnaryFunctions.scala", "content": "package frameless\npackage functions\n\nimport org.apache.spark.sql.{Column, functions => sparkFunctions}\n\nimport scala.math.Ordering\n\ntrait UnaryFunctions {\n /** Returns length of array\n *\n * apache/spark\n */\n def size[T, A, V[_] : CatalystSizableCollection](column: TypedColumn[T, V[A]]): TypedColumn[T, Int] =\n new TypedColumn[T, Int](implicitly[CatalystSizableCollection[V]].sizeOp(column.untyped))\n\n /** Returns length of Map\n *\n * apache/spark\n */\n def size[T, A, B](column: TypedColumn[T, Map[A, B]]): TypedColumn[T, Int] =\n new TypedColumn[T, Int](sparkFunctions.size(column.untyped))\n\n /** Sorts the input array for the given column in ascending order, according to\n * the natural ordering of the array elements.\n *\n * apache/spark\n */\n def sortAscending[T, A: Ordering, V[_] : CatalystSortableCollection](column: TypedColumn[T, V[A]]): TypedColumn[T, V[A]] =\n new TypedColumn[T, V[A]](implicitly[CatalystSortableCollection[V]].sortOp(column.untyped, sortAscending = true))(column.uencoder)\n\n /** Sorts the input array for the given column in descending order, according to\n * the natural ordering of the array elements.\n *\n * apache/spark\n */\n def sortDescending[T, A: Ordering, V[_] : CatalystSortableCollection](column: TypedColumn[T, V[A]]): TypedColumn[T, V[A]] =\n new TypedColumn[T, V[A]](implicitly[CatalystSortableCollection[V]].sortOp(column.untyped, sortAscending = false))(column.uencoder)\n\n\n /** Creates a new row for each element in the given collection. The column types\n * eligible for this operation are constrained by CatalystExplodableCollection.\n *\n * apache/spark\n */\n @deprecated(\"Use explode() from the TypedDataset instead. This method will result in \" +\n \"runtime error if applied to two columns in the same select statement.\", \"0.6.2\")\n def explode[T, A: TypedEncoder, V[_] : CatalystExplodableCollection](column: TypedColumn[T, V[A]]): TypedColumn[T, A] =\n new TypedColumn[T, A](sparkFunctions.explode(column.untyped))\n}\n\ntrait CatalystSizableCollection[V[_]] {\n def sizeOp(col: Column): Column\n}\n\nobject CatalystSizableCollection {\n implicit def sizableVector: CatalystSizableCollection[Vector] = new CatalystSizableCollection[Vector] {\n def sizeOp(col: Column): Column = sparkFunctions.size(col)\n }\n\n implicit def sizableArray: CatalystSizableCollection[Array] = new CatalystSizableCollection[Array] {\n def sizeOp(col: Column): Column = sparkFunctions.size(col)\n }\n\n implicit def sizableList: CatalystSizableCollection[List] = new CatalystSizableCollection[List] {\n def sizeOp(col: Column): Column = sparkFunctions.size(col)\n }\n\n}\n\ntrait CatalystExplodableCollection[V[_]]\n\nobject CatalystExplodableCollection {\n implicit def explodableVector: CatalystExplodableCollection[Vector] = new CatalystExplodableCollection[Vector] {}\n implicit def explodableArray: CatalystExplodableCollection[Array] = new CatalystExplodableCollection[Array] {}\n implicit def explodableList: CatalystExplodableCollection[List] = new CatalystExplodableCollection[List] {}\n implicit def explodableSeq: CatalystExplodableCollection[Seq] = new CatalystExplodableCollection[Seq] {}\n}\n\ntrait CatalystSortableCollection[V[_]] {\n def sortOp(col: Column, sortAscending: Boolean): Column\n}\n\nobject CatalystSortableCollection {\n implicit def sortableVector: CatalystSortableCollection[Vector] = new CatalystSortableCollection[Vector] {\n def sortOp(col: Column, sortAscending: Boolean): Column = sparkFunctions.sort_array(col, sortAscending)\n }\n\n implicit def sortableArray: CatalystSortableCollection[Array] = new CatalystSortableCollection[Array] {\n def sortOp(col: Column, sortAscending: Boolean): Column = sparkFunctions.sort_array(col, sortAscending)\n }\n\n implicit def sortableList: CatalystSortableCollection[List] = new CatalystSortableCollection[List] {\n def sortOp(col: Column, sortAscending: Boolean): Column = sparkFunctions.sort_array(col, sortAscending)\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/functions/package.scala", "content": "package frameless\n\nimport scala.reflect.ClassTag\n\nimport shapeless._\nimport shapeless.labelled.FieldType\nimport shapeless.ops.hlist.IsHCons\nimport shapeless.ops.record.{ Keys, Values }\n\nimport org.apache.spark.sql.{ reflection => ScalaReflection }\nimport org.apache.spark.sql.catalyst.expressions.Literal\n\npackage object functions extends Udf with UnaryFunctions {\n\n object aggregate extends AggregateFunctions\n object nonAggregate extends NonAggregateFunctions\n\n /**\n * Creates a [[frameless.TypedAggregate]] of literal value. If A is to be encoded using an Injection make\n * sure the injection instance is in scope.\n *\n * apache/spark\n */\n def litAggr[A, T](\n value: A\n )(implicit\n i0: TypedEncoder[A],\n i1: Refute[IsValueClass[A]]\n ): TypedAggregate[T, A] =\n new TypedAggregate[T, A](lit(value).expr)\n\n /**\n * Creates a [[frameless.TypedColumn]] of literal value. If A is to be encoded using an Injection make\n * sure the injection instance is in scope.\n *\n * apache/spark\n *\n * @tparam A the literal value type\n * @tparam T the row type\n */\n def lit[A, T](\n value: A\n )(implicit\n encoder: TypedEncoder[A]\n ): TypedColumn[T, A] = {\n\n if (\n ScalaReflection.isNativeType(\n encoder.jvmRepr\n ) && encoder.catalystRepr == encoder.jvmRepr\n ) {\n val expr = Literal(value, encoder.catalystRepr)\n\n new TypedColumn(expr)\n } else {\n val expr = new Literal(value, encoder.jvmRepr)\n\n new TypedColumn[T, A](\n Lit(\n dataType = encoder.catalystRepr,\n nullable = encoder.nullable,\n show = () => value.toString,\n catalystExpr = encoder.toCatalyst(expr)\n )\n )\n }\n }\n\n /**\n * Creates a [[frameless.TypedColumn]] of literal value\n * for a Value class `A`.\n *\n * @tparam A the value class\n * @tparam T the row type\n */\n def litValue[\n A: IsValueClass,\n T,\n G <: ::[_, HNil],\n H <: ::[_ <: FieldType[_ <: Symbol, _], HNil],\n K <: Symbol,\n V,\n KS <: ::[_ <: Symbol, HNil],\n VS <: HList\n ](value: A\n )(implicit\n i0: LabelledGeneric.Aux[A, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil],\n i3: Keys.Aux[H, KS],\n i4: Values.Aux[H, VS],\n i5: IsHCons.Aux[KS, K, HNil],\n i6: IsHCons.Aux[VS, V, HNil],\n i7: TypedEncoder[V],\n i8: ClassTag[A]\n ): TypedColumn[T, A] = {\n val expr = {\n val field: H = i1(i0.to(value))\n val v: V = i6.head(i4(field))\n\n new Literal(v, i7.jvmRepr)\n }\n\n implicit val enc: TypedEncoder[A] =\n RecordFieldEncoder.valueClass[A, G, H, K, V, KS].encoder\n\n new TypedColumn[T, A](\n Lit(\n dataType = i7.catalystRepr,\n nullable = i7.nullable,\n show = () => value.toString,\n i7.toCatalyst(expr)\n )\n )\n }\n\n /**\n * Creates a [[frameless.TypedColumn]] of literal value\n * for an optional Value class `A`.\n *\n * @tparam A the value class\n * @tparam T the row type\n */\n def litValue[\n A: IsValueClass,\n T,\n G <: ::[_, HNil],\n H <: ::[_ <: FieldType[_ <: Symbol, _], HNil],\n K <: Symbol,\n V,\n KS <: ::[_ <: Symbol, HNil],\n VS <: HList\n ](value: Option[A]\n )(implicit\n i0: LabelledGeneric.Aux[A, G],\n i1: DropUnitValues.Aux[G, H],\n i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil],\n i3: Keys.Aux[H, KS],\n i4: Values.Aux[H, VS],\n i5: IsHCons.Aux[KS, K, HNil],\n i6: IsHCons.Aux[VS, V, HNil],\n i7: TypedEncoder[V],\n i8: ClassTag[A]\n ): TypedColumn[T, Option[A]] = {\n val expr = value match {\n case Some(some) => {\n val field: H = i1(i0.to(some))\n val v: V = i6.head(i4(field))\n\n new Literal(v, i7.jvmRepr)\n }\n\n case _ =>\n Literal.create(null, i7.jvmRepr)\n }\n\n implicit val enc: TypedEncoder[A] =\n RecordFieldEncoder.valueClass[A, G, H, K, V, KS].encoder\n\n new TypedColumn[T, Option[A]](\n Lit(\n dataType = i7.catalystRepr,\n nullable = true,\n show = () => value.toString,\n i7.toCatalyst(expr)\n )\n )\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/AggregateTypes.scala", "content": "package frameless\npackage ops\n\nimport shapeless._\n\n/** A type class to extract the column types out of an HList of [[frameless.TypedAggregate]].\n *\n * @note This type class is mostly a workaround to issue with slow implicit derivation for Comapped.\n * @example\n * {{{\n * type U = TypedAggregate[T,A] :: TypedAggregate[T,B] :: TypedAggregate[T,C] :: HNil\n * type Out = A :: B :: C :: HNil\n * }}}\n */\ntrait AggregateTypes[V, U <: HList] {\n type Out <: HList\n}\n\nobject AggregateTypes {\n type Aux[V, U <: HList, Out0 <: HList] = AggregateTypes[V, U] {type Out = Out0}\n\n implicit def deriveHNil[T]: AggregateTypes.Aux[T, HNil, HNil] = new AggregateTypes[T, HNil] { type Out = HNil }\n\n implicit def deriveCons1[T, H, TT <: HList, V <: HList](\n implicit tail: AggregateTypes.Aux[T, TT, V]\n ): AggregateTypes.Aux[T, TypedAggregate[T, H] :: TT, H :: V] =\n new AggregateTypes[T, TypedAggregate[T, H] :: TT] {type Out = H :: V}\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/As.scala", "content": "package frameless\npackage ops\n\nimport shapeless.{::, Generic, HList, Lazy}\n\n/** Evidence for correctness of `TypedDataset[T].as[U]` */\nclass As[T, U] private (implicit val encoder: TypedEncoder[U])\n\nobject As extends LowPriorityAs {\n\n final class Equiv[A, B] private[ops] ()\n\n implicit def equivIdentity[A] = new Equiv[A, A]\n\n implicit def deriveAs[A, B]\n (implicit\n i0: TypedEncoder[B],\n i1: Equiv[A, B]\n ): As[A, B] = new As[A, B]\n\n}\n\ntrait LowPriorityAs {\n\n import As.Equiv\n\n implicit def equivHList[AH, AT <: HList, BH, BT <: HList]\n (implicit\n i0: Lazy[Equiv[AH, BH]],\n i1: Equiv[AT, BT]\n ): Equiv[AH :: AT, BH :: BT] = new Equiv[AH :: AT, BH :: BT]\n\n implicit def equivGeneric[A, B, R, S]\n (implicit\n i0: Generic.Aux[A, R],\n i1: Generic.Aux[B, S],\n i2: Lazy[Equiv[R, S]]\n ): Equiv[A, B] = new Equiv[A, B]\n\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/ColumnTypes.scala", "content": "package frameless\npackage ops\n\nimport shapeless._\n\n/** A type class to extract the column types out of an HList of [[frameless.TypedColumn]].\n *\n * @note This type class is mostly a workaround to issue with slow implicit derivation for Comapped.\n * @example\n * {{{\n * type U = TypedColumn[T,A] :: TypedColumn[T,B] :: TypedColumn[T,C] :: HNil\n * type Out = A :: B :: C :: HNil\n * }}}\n */\ntrait ColumnTypes[T, U <: HList] {\n type Out <: HList\n}\n\nobject ColumnTypes {\n type Aux[T, U <: HList, Out0 <: HList] = ColumnTypes[T, U] {type Out = Out0}\n\n implicit def deriveHNil[T]: ColumnTypes.Aux[T, HNil, HNil] = new ColumnTypes[T, HNil] { type Out = HNil }\n\n implicit def deriveCons[T, H, TT <: HList, V <: HList](\n implicit tail: ColumnTypes.Aux[T, TT, V]\n ): ColumnTypes.Aux[T, TypedColumn[T, H] :: TT, H :: V] =\n new ColumnTypes[T, TypedColumn[T, H] :: TT] {type Out = H :: V}\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/GroupByOps.scala", "content": "package frameless\npackage ops\n\nimport org.apache.spark.sql.catalyst.analysis.UnresolvedAlias\nimport org.apache.spark.sql.catalyst.plans.logical.Project\nimport org.apache.spark.sql.{Column, Dataset, FramelessInternals, RelationalGroupedDataset}\nimport shapeless._\nimport shapeless.ops.hlist.{Length, Mapped, Prepend, ToList, ToTraversable, Tupler}\n\nclass GroupedByManyOps[T, TK <: HList, K <: HList, KT]\n (self: TypedDataset[T], groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: ToTraversable.Aux[TK, List, UntypedExpression[T]],\n i3: Tupler.Aux[K, KT]\n ) extends AggregatingOps[T, TK, K, KT](self, groupedBy, (dataset, cols) => dataset.groupBy(cols: _*)) {\n object agg extends ProductArgs {\n def applyProduct[TC <: HList, C <: HList, Out0 <: HList, Out1]\n (columns: TC)\n (implicit\n i3: AggregateTypes.Aux[T, TC, C],\n i4: Prepend.Aux[K, C, Out0],\n i5: Tupler.Aux[Out0, Out1],\n i6: TypedEncoder[Out1],\n i7: ToTraversable.Aux[TC, List, UntypedExpression[T]]\n ): TypedDataset[Out1] = {\n aggregate[TC, Out1](columns)\n }\n }\n}\n\nclass GroupedBy1Ops[K1, V](\n self: TypedDataset[V],\n g1: TypedColumn[V, K1]\n) {\n private def underlying = new GroupedByManyOps(self, g1 :: HNil)\n private implicit def eg1 = g1.uencoder\n\n def agg[U1](c1: TypedAggregate[V, U1]): TypedDataset[(K1, U1)] = {\n implicit val e1 = c1.uencoder\n underlying.agg(c1)\n }\n\n def agg[U1, U2](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2]): TypedDataset[(K1, U1, U2)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder\n underlying.agg(c1, c2)\n }\n\n def agg[U1, U2, U3](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3]): TypedDataset[(K1, U1, U2, U3)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder\n underlying.agg(c1, c2, c3)\n }\n\n def agg[U1, U2, U3, U4](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4]): TypedDataset[(K1, U1, U2, U3, U4)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder\n underlying.agg(c1, c2, c3, c4)\n }\n\n def agg[U1, U2, U3, U4, U5](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4], c5: TypedAggregate[V, U5]): TypedDataset[(K1, U1, U2, U3, U4, U5)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder; implicit val e5 = c5.uencoder\n underlying.agg(c1, c2, c3, c4, c5)\n }\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n */\n object deserialized {\n def mapGroups[U: TypedEncoder](f: (K1, Iterator[V]) => U): TypedDataset[U] = {\n underlying.deserialized.mapGroups(AggregatingOps.tuple1(f))\n }\n\n def flatMapGroups[U: TypedEncoder](f: (K1, Iterator[V]) => TraversableOnce[U]): TypedDataset[U] = {\n underlying.deserialized.flatMapGroups(AggregatingOps.tuple1(f))\n }\n }\n\n def pivot[P: CatalystPivotable](pivotColumn: TypedColumn[V, P]): PivotNotValues[V, TypedColumn[V,K1] :: HNil, P] =\n PivotNotValues(self, g1 :: HNil, pivotColumn)\n}\n\n\nclass GroupedBy2Ops[K1, K2, V](\n self: TypedDataset[V],\n g1: TypedColumn[V, K1],\n g2: TypedColumn[V, K2]\n) {\n private def underlying = new GroupedByManyOps(self, g1 :: g2 :: HNil)\n private implicit def eg1 = g1.uencoder\n private implicit def eg2 = g2.uencoder\n\n def agg[U1](c1: TypedAggregate[V, U1]): TypedDataset[(K1, K2, U1)] = {\n implicit val e1 = c1.uencoder\n underlying.agg(c1)\n }\n\n def agg[U1, U2](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2]): TypedDataset[(K1, K2, U1, U2)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder\n underlying.agg(c1, c2)\n }\n\n def agg[U1, U2, U3](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3]): TypedDataset[(K1, K2, U1, U2, U3)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder\n underlying.agg(c1, c2, c3)\n }\n\n def agg[U1, U2, U3, U4](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4]): TypedDataset[(K1, K2, U1, U2, U3, U4)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder\n underlying.agg(c1 , c2 , c3 , c4)\n }\n\n def agg[U1, U2, U3, U4, U5](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4], c5: TypedAggregate[V, U5]): TypedDataset[(K1, K2, U1, U2, U3, U4, U5)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder; implicit val e5 = c5.uencoder\n underlying.agg(c1, c2, c3, c4, c5)\n }\n\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n */\n object deserialized {\n def mapGroups[U: TypedEncoder](f: ((K1, K2), Iterator[V]) => U): TypedDataset[U] = {\n underlying.deserialized.mapGroups(f)\n }\n\n def flatMapGroups[U: TypedEncoder](f: ((K1, K2), Iterator[V]) => TraversableOnce[U]): TypedDataset[U] = {\n underlying.deserialized.flatMapGroups(f)\n }\n }\n\n def pivot[P: CatalystPivotable](pivotColumn: TypedColumn[V, P]):\n PivotNotValues[V, TypedColumn[V,K1] :: TypedColumn[V, K2] :: HNil, P] =\n PivotNotValues(self, g1 :: g2 :: HNil, pivotColumn)\n}\n\nprivate[ops] abstract class AggregatingOps[T, TK <: HList, K <: HList, KT]\n (self: TypedDataset[T], groupedBy: TK, groupingFunc: (Dataset[T], Seq[Column]) => RelationalGroupedDataset)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: ToTraversable.Aux[TK, List, UntypedExpression[T]],\n i2: Tupler.Aux[K, KT]\n ) {\n def aggregate[TC <: HList, Out1](columns: TC)\n (implicit\n i7: TypedEncoder[Out1],\n i8: ToTraversable.Aux[TC, List, UntypedExpression[T]]\n ): TypedDataset[Out1] = {\n def expr(c: UntypedExpression[T]): Column = new Column(c.expr)\n\n val groupByExprs = groupedBy.toList[UntypedExpression[T]].map(expr)\n val aggregates =\n if (retainGroupColumns) columns.toList[UntypedExpression[T]].map(expr)\n else groupByExprs ++ columns.toList[UntypedExpression[T]].map(expr)\n\n val aggregated =\n groupingFunc(self.dataset, groupByExprs)\n .agg(aggregates.head, aggregates.tail: _*)\n .as[Out1](TypedExpressionEncoder[Out1])\n\n TypedDataset.create[Out1](aggregated)\n }\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n */\n object deserialized {\n def mapGroups[U: TypedEncoder](\n f: (KT, Iterator[T]) => U\n )(implicit e: TypedEncoder[KT]): TypedDataset[U] = {\n val func = (key: KT, it: Iterator[T]) => Iterator(f(key, it))\n flatMapGroups(func)\n }\n\n def flatMapGroups[U: TypedEncoder](\n f: (KT, Iterator[T]) => TraversableOnce[U]\n )(implicit e: TypedEncoder[KT]): TypedDataset[U] = {\n implicit val tendcoder = self.encoder\n\n val cols = groupedBy.toList[UntypedExpression[T]]\n val logicalPlan = FramelessInternals.logicalPlan(self.dataset)\n val withKeyColumns = logicalPlan.output ++ cols.map(_.expr).map(UnresolvedAlias(_))\n val withKey = Project(withKeyColumns, logicalPlan)\n val executed = FramelessInternals.executePlan(self.dataset, withKey)\n val keyAttributes = executed.analyzed.output.takeRight(cols.size)\n val dataAttributes = executed.analyzed.output.dropRight(cols.size)\n\n val mapGroups = MapGroups(\n f,\n keyAttributes,\n dataAttributes,\n executed.analyzed\n )(TypedExpressionEncoder[KT], TypedExpressionEncoder[T], TypedExpressionEncoder[U])\n\n val groupedAndFlatMapped = FramelessInternals.mkDataset(\n self.dataset.sqlContext,\n mapGroups,\n TypedExpressionEncoder[U]\n )\n\n TypedDataset.create(groupedAndFlatMapped)\n }\n }\n\n private def retainGroupColumns: Boolean = {\n self.dataset.sqlContext.getConf(\"spark.sql.retainGroupColumns\", \"true\").toBoolean\n }\n\n def pivot[P: CatalystPivotable](pivotColumn: TypedColumn[T, P]): PivotNotValues[T, TK, P] =\n PivotNotValues(self, groupedBy, pivotColumn)\n}\n\nprivate[ops] object AggregatingOps {\n /** Utility function to help Spark with serialization of closures */\n def tuple1[K1, V, U](f: (K1, Iterator[V]) => U): (Tuple1[K1], Iterator[V]) => U = {\n (x: Tuple1[K1], it: Iterator[V]) => f(x._1, it)\n }\n}\n\n/** Represents a typed Pivot operation.\n */\nfinal case class Pivot[T, GroupedColumns <: HList, PivotType, Values <: HList](\n ds: TypedDataset[T],\n groupedBy: GroupedColumns,\n pivotedBy: TypedColumn[T, PivotType],\n values: Values\n) {\n\n object agg extends ProductArgs {\n def applyProduct[AggrColumns <: HList, AggrColumnTypes <: HList, GroupedColumnTypes <: HList, NumValues <: Nat, TypesForPivotedValues <: HList, TypesForPivotedValuesOpt <: HList, OutAsHList <: HList, Out]\n (aggrColumns: AggrColumns)\n (implicit\n i0: AggregateTypes.Aux[T, AggrColumns, AggrColumnTypes],\n i1: ColumnTypes.Aux[T, GroupedColumns, GroupedColumnTypes],\n i2: Length.Aux[Values, NumValues],\n i3: Repeat.Aux[AggrColumnTypes, NumValues, TypesForPivotedValues],\n i4: Mapped.Aux[TypesForPivotedValues, Option, TypesForPivotedValuesOpt],\n i5: Prepend.Aux[GroupedColumnTypes, TypesForPivotedValuesOpt, OutAsHList],\n i6: Tupler.Aux[OutAsHList, Out],\n i7: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n def mapAny[X](h: HList)(f: Any => X): List[X] =\n h match {\n case HNil => Nil\n case x :: xs => f(x) :: mapAny(xs)(f)\n }\n\n val aggCols: Seq[Column] = mapAny(aggrColumns)(x => new Column(x.asInstanceOf[TypedAggregate[_,_]].expr))\n val tmp = ds.dataset.toDF()\n .groupBy(mapAny(groupedBy)(_.asInstanceOf[TypedColumn[_, _]].untyped): _*)\n .pivot(pivotedBy.untyped.toString, mapAny(values)(identity))\n .agg(aggCols.head, aggCols.tail:_*)\n .as[Out](TypedExpressionEncoder[Out])\n TypedDataset.create(tmp)\n }\n }\n}\n\nfinal case class PivotNotValues[T, GroupedColumns <: HList, PivotType](\n ds: TypedDataset[T],\n groupedBy: GroupedColumns,\n pivotedBy: TypedColumn[T, PivotType]\n) extends ProductArgs {\n\n def onProduct[Values <: HList](values: Values)(\n implicit validValues: ToList[Values, PivotType] // validValues: FilterNot.Aux[Values, PivotType, HNil] // did not work\n ): Pivot[T, GroupedColumns, PivotType, Values] = Pivot(ds, groupedBy, pivotedBy, values)\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/RelationalGroupsOps.scala", "content": "package frameless\npackage ops\n\nimport org.apache.spark.sql.{Column, Dataset, RelationalGroupedDataset}\nimport shapeless.ops.hlist.{Mapped, Prepend, ToTraversable, Tupler}\nimport shapeless.{::, HList, HNil, ProductArgs}\n\n/**\n * @param groupingFunc functions used to group elements, can be cube or rollup\n * @tparam T the original `TypedDataset's` type T\n * @tparam TK all columns chosen for aggregation\n * @tparam K individual columns' types as HList\n * @tparam KT individual columns' types as Tuple\n */\nprivate[ops] abstract class RelationalGroupsOps[T, TK <: HList, K <: HList, KT]\n (self: TypedDataset[T], groupedBy: TK, groupingFunc: (Dataset[T], Seq[Column]) => RelationalGroupedDataset)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: ToTraversable.Aux[TK, List, UntypedExpression[T]],\n i2: Tupler.Aux[K, KT]\n ) extends AggregatingOps(self, groupedBy, groupingFunc){\n\n object agg extends ProductArgs {\n /**\n * @tparam TC resulting columns after aggregation function\n * @tparam C individual columns' types as HList\n * @tparam OptK columns' types mapped to Option\n * @tparam Out0 OptK columns appended to C\n * @tparam Out1 output type\n */\n def applyProduct[TC <: HList, C <: HList, OptK <: HList, Out0 <: HList, Out1]\n (columns: TC)\n (implicit\n i3: AggregateTypes.Aux[T, TC, C], // shares individual columns' types after agg function as HList\n i4: Mapped.Aux[K, Option, OptK], // maps all original columns' types to Option\n i5: Prepend.Aux[OptK, C, Out0], // concatenates Option columns with those resulting from applying agg function\n i6: Tupler.Aux[Out0, Out1], // converts resulting HList into Tuple for output type\n i7: TypedEncoder[Out1], // proof that there is `TypedEncoder` for the output type\n i8: ToTraversable.Aux[TC, List, UntypedExpression[T]] // allows converting this HList to ordinary List\n ): TypedDataset[Out1] = {\n aggregate[TC, Out1](columns)\n }\n }\n}\n\nprivate[ops] abstract class RelationalGroups1Ops[K1, V](self: TypedDataset[V], g1: TypedColumn[V, K1]) {\n protected def underlying: RelationalGroupsOps[V, ::[TypedColumn[V, K1], HNil], ::[K1, HNil], Tuple1[K1]]\n private implicit def eg1 = g1.uencoder\n\n def agg[U1](c1: TypedAggregate[V, U1]): TypedDataset[(Option[K1], U1)] = {\n implicit val e1 = c1.uencoder\n underlying.agg(c1)\n }\n\n def agg[U1, U2](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2]): TypedDataset[(Option[K1], U1, U2)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder\n underlying.agg(c1, c2)\n }\n\n def agg[U1, U2, U3](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3]): TypedDataset[(Option[K1], U1, U2, U3)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder\n underlying.agg(c1, c2, c3)\n }\n\n def agg[U1, U2, U3, U4](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4]): TypedDataset[(Option[K1], U1, U2, U3, U4)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder\n underlying.agg(c1, c2, c3, c4)\n }\n\n def agg[U1, U2, U3, U4, U5](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4], c5: TypedAggregate[V, U5]): TypedDataset[(Option[K1], U1, U2, U3, U4, U5)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder; implicit val e5 = c5.uencoder\n underlying.agg(c1, c2, c3, c4, c5)\n }\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n */\n object deserialized {\n def mapGroups[U: TypedEncoder](f: (K1, Iterator[V]) => U): TypedDataset[U] = {\n underlying.deserialized.mapGroups(AggregatingOps.tuple1(f))\n }\n\n def flatMapGroups[U: TypedEncoder](f: (K1, Iterator[V]) => TraversableOnce[U]): TypedDataset[U] = {\n underlying.deserialized.flatMapGroups(AggregatingOps.tuple1(f))\n }\n }\n\n def pivot[P: CatalystPivotable](pivotColumn: TypedColumn[V, P]): PivotNotValues[V, TypedColumn[V,K1] :: HNil, P] =\n PivotNotValues(self, g1 :: HNil, pivotColumn)\n}\n\nprivate[ops] abstract class RelationalGroups2Ops[K1, K2, V](self: TypedDataset[V], g1: TypedColumn[V, K1], g2: TypedColumn[V, K2]) {\n protected def underlying: RelationalGroupsOps[V, ::[TypedColumn[V, K1], ::[TypedColumn[V, K2], HNil]], ::[K1, ::[K2, HNil]], (K1, K2)]\n private implicit def eg1 = g1.uencoder\n private implicit def eg2 = g2.uencoder\n\n def agg[U1](c1: TypedAggregate[V, U1]): TypedDataset[(Option[K1], Option[K2], U1)] = {\n implicit val e1 = c1.uencoder\n underlying.agg(c1)\n }\n\n def agg[U1, U2](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2]): TypedDataset[(Option[K1], Option[K2], U1, U2)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder\n underlying.agg(c1, c2)\n }\n\n def agg[U1, U2, U3](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3]): TypedDataset[(Option[K1], Option[K2], U1, U2, U3)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder\n underlying.agg(c1, c2, c3)\n }\n\n def agg[U1, U2, U3, U4](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4]): TypedDataset[(Option[K1], Option[K2], U1, U2, U3, U4)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder\n underlying.agg(c1 , c2 , c3 , c4)\n }\n\n def agg[U1, U2, U3, U4, U5](c1: TypedAggregate[V, U1], c2: TypedAggregate[V, U2], c3: TypedAggregate[V, U3], c4: TypedAggregate[V, U4], c5: TypedAggregate[V, U5]): TypedDataset[(Option[K1], Option[K2], U1, U2, U3, U4, U5)] = {\n implicit val e1 = c1.uencoder; implicit val e2 = c2.uencoder; implicit val e3 = c3.uencoder; implicit val e4 = c4.uencoder; implicit val e5 = c5.uencoder\n underlying.agg(c1, c2, c3, c4, c5)\n }\n\n /** Methods on `TypedDataset[T]` that go through a full serialization and\n * deserialization of `T`, and execute outside of the Catalyst runtime.\n */\n object deserialized {\n def mapGroups[U: TypedEncoder](f: ((K1, K2), Iterator[V]) => U): TypedDataset[U] = {\n underlying.deserialized.mapGroups(f)\n }\n\n def flatMapGroups[U: TypedEncoder](f: ((K1, K2), Iterator[V]) => TraversableOnce[U]): TypedDataset[U] = {\n underlying.deserialized.flatMapGroups(f)\n }\n }\n\n def pivot[P: CatalystPivotable](pivotColumn: TypedColumn[V, P]):\n PivotNotValues[V, TypedColumn[V,K1] :: TypedColumn[V, K2] :: HNil, P] =\n PivotNotValues(self, g1 :: g2 :: HNil, pivotColumn)\n}\n\nclass RollupManyOps[T, TK <: HList, K <: HList, KT](self: TypedDataset[T], groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: ToTraversable.Aux[TK, List, UntypedExpression[T]],\n i2: Tupler.Aux[K, KT]\n ) extends RelationalGroupsOps[T, TK, K, KT](self, groupedBy, (dataset, cols) => dataset.rollup(cols: _*))\n\nclass Rollup1Ops[K1, V](self: TypedDataset[V], g1: TypedColumn[V, K1]) extends RelationalGroups1Ops(self, g1) {\n override protected def underlying = new RollupManyOps(self, g1 :: HNil)\n}\n\nclass Rollup2Ops[K1, K2, V](self: TypedDataset[V], g1: TypedColumn[V, K1], g2: TypedColumn[V, K2]) extends RelationalGroups2Ops(self, g1, g2) {\n override protected def underlying = new RollupManyOps(self, g1 :: g2 :: HNil)\n}\n\nclass CubeManyOps[T, TK <: HList, K <: HList, KT](self: TypedDataset[T], groupedBy: TK)\n (implicit\n i0: ColumnTypes.Aux[T, TK, K],\n i1: ToTraversable.Aux[TK, List, UntypedExpression[T]],\n i2: Tupler.Aux[K, KT]\n ) extends RelationalGroupsOps[T, TK, K, KT](self, groupedBy, (dataset, cols) => dataset.cube(cols: _*))\n\nclass Cube1Ops[K1, V](self: TypedDataset[V], g1: TypedColumn[V, K1]) extends RelationalGroups1Ops(self, g1) {\n override protected def underlying = new CubeManyOps(self, g1 :: HNil)\n}\n\nclass Cube2Ops[K1, K2, V](self: TypedDataset[V], g1: TypedColumn[V, K1], g2: TypedColumn[V, K2]) extends RelationalGroups2Ops(self, g1, g2) {\n override protected def underlying = new CubeManyOps(self, g1 :: g2 :: HNil)\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/Repeat.scala", "content": "package frameless\npackage ops\n\nimport shapeless.{HList, Nat, Succ}\nimport shapeless.ops.hlist.Prepend\n\n/** Typeclass supporting repeating L-typed HLists N times.\n *\n * Repeat[Int :: String :: HNil, Nat._2].Out =:=\n * Int :: String :: Int :: String :: HNil\n *\n * By Jeremy Smith. To be replaced by `shapeless.ops.hlists.Repeat`\n * once (https://github.com/milessabin/shapeless/pull/730 is published.\n */\ntrait Repeat[L <: HList, N <: Nat] {\n type Out <: HList\n}\n\nobject Repeat {\n type Aux[L <: HList, N <: Nat, Out0 <: HList] = Repeat[L, N] { type Out = Out0 }\n\n implicit def base[L <: HList]: Aux[L, Nat._1, L] = new Repeat[L, Nat._1] {\n type Out = L\n }\n\n implicit def succ[L <: HList, Prev <: Nat, PrevOut <: HList, P <: HList]\n (implicit\n i0: Aux[L, Prev, PrevOut],\n i1: Prepend.Aux[L, PrevOut, P]\n ): Aux[L, Succ[Prev], P] = new Repeat[L, Succ[Prev]] {\n type Out = P\n }\n}\n" }, { "path": "dataset/src/main/scala/frameless/ops/SmartProject.scala", "content": "package frameless\npackage ops\n\nimport shapeless.ops.hlist.ToTraversable\nimport shapeless.ops.record.{Keys, SelectAll, Values}\nimport shapeless.{HList, LabelledGeneric}\n\nimport scala.annotation.implicitNotFound\n\n@implicitNotFound(msg = \"Cannot prove that ${T} can be projected to ${U}. Perhaps not all member names and types of ${U} are the same in ${T}?\")\ncase class SmartProject[T: TypedEncoder, U: TypedEncoder](apply: TypedDataset[T] => TypedDataset[U])\n\nobject SmartProject {\n /**\n * Proofs that there is a type-safe projection from a type T to another type U. It requires that:\n * (a) both T and U are Products for which a LabelledGeneric can be derived (e.g., case classes),\n * (b) all members of U have a corresponding member in T that has both the same name and type.\n *\n * @param i0 the LabelledGeneric derived for T\n * @param i1 the LabelledGeneric derived for U\n * @param i2 the keys of U\n * @param i3 selects all the values from T using the keys of U\n * @param i4 selects all the values of LabeledGeneric[U]\n * @param i5 proof that U and the projection of T have the same type\n * @param i6 allows for traversing the keys of U\n * @tparam T the original type T\n * @tparam U the projected type U\n * @tparam TRec shapeless' Record representation of T\n * @tparam TProj the projection of T using the keys of U\n * @tparam URec shapeless' Record representation of U\n * @tparam UVals the values of U as an HList\n * @tparam UKeys the keys of U as an HList\n * @return a projection if it exists\n */\n implicit def deriveProduct[T: TypedEncoder, U: TypedEncoder, TRec <: HList, TProj <: HList, URec <: HList, UVals <: HList, UKeys <: HList]\n (implicit\n i0: LabelledGeneric.Aux[T, TRec],\n i1: LabelledGeneric.Aux[U, URec],\n i2: Keys.Aux[URec, UKeys],\n i3: SelectAll.Aux[TRec, UKeys, TProj],\n i4: Values.Aux[URec, UVals],\n i5: UVals =:= TProj,\n i6: ToTraversable.Aux[UKeys, Seq, Symbol]\n ): SmartProject[T,U] = SmartProject[T, U]({ from =>\n val names = implicitly[Keys.Aux[URec, UKeys]].apply().to[Seq].map(_.name).map(from.dataset.col)\n TypedDataset.create(from.dataset.toDF().select(names: _*).as[U](TypedExpressionEncoder[U]))\n })\n}\n" }, { "path": "dataset/src/main/scala/frameless/syntax/package.scala", "content": "package frameless\n\npackage object syntax extends FramelessSyntax {\n implicit val DefaultSparkDelay: SparkDelay[Job] = Job.framelessSparkDelayForJob\n}\n" }, { "path": "dataset/src/main/scala/org/apache/spark/sql/FramelessInternals.scala", "content": "package org.apache.spark.sql\n\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.catalyst.expressions.codegen._\nimport org.apache.spark.sql.catalyst.expressions.{Alias, CreateStruct}\nimport org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression}\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.catalyst.plans.logical.LogicalPlan\nimport org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}\nimport org.apache.spark.sql.execution.QueryExecution\nimport org.apache.spark.sql.types._\nimport org.apache.spark.sql.types.ObjectType\nimport scala.reflect.ClassTag\n\nobject FramelessInternals {\n def objectTypeFor[A](implicit classTag: ClassTag[A]): ObjectType = ObjectType(classTag.runtimeClass)\n\n def resolveExpr(ds: Dataset[_], colNames: Seq[String]): NamedExpression = {\n ds.toDF().queryExecution.analyzed.resolve(colNames, ds.sparkSession.sessionState.analyzer.resolver).getOrElse {\n throw new AnalysisException(\n s\"\"\"Cannot resolve column name \"$colNames\" among (${ds.schema.fieldNames.mkString(\", \")})\"\"\")\n }\n }\n\n def expr(column: Column): Expression = column.expr\n\n def logicalPlan(ds: Dataset[_]): LogicalPlan = ds.logicalPlan\n\n def executePlan(ds: Dataset[_], plan: LogicalPlan): QueryExecution =\n ds.sparkSession.sessionState.executePlan(plan)\n\n def joinPlan(ds: Dataset[_], plan: LogicalPlan, leftPlan: LogicalPlan, rightPlan: LogicalPlan): LogicalPlan = {\n val joined = executePlan(ds, plan)\n val leftOutput = joined.analyzed.output.take(leftPlan.output.length)\n val rightOutput = joined.analyzed.output.takeRight(rightPlan.output.length)\n\n Project(List(\n Alias(CreateStruct(leftOutput), \"_1\")(),\n Alias(CreateStruct(rightOutput), \"_2\")()\n ), joined.analyzed)\n }\n\n def mkDataset[T](sqlContext: SQLContext, plan: LogicalPlan, encoder: Encoder[T]): Dataset[T] =\n new Dataset(sqlContext, plan, encoder)\n\n def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame =\n Dataset.ofRows(sparkSession, logicalPlan)\n\n // because org.apache.spark.sql.types.UserDefinedType is private[spark]\n type UserDefinedType[A >: Null] = org.apache.spark.sql.types.UserDefinedType[A]\n\n // below only tested in SelfJoinTests.colLeft and colRight are equivalent to col outside of joins\n // - via files (codegen) forces doGenCode eval.\n /** Expression to tag columns from the left hand side of join expression. */\n case class DisambiguateLeft[T](tagged: Expression) extends Expression with NonSQLExpression {\n def eval(input: InternalRow): Any = tagged.eval(input)\n def nullable: Boolean = false\n def children: Seq[Expression] = tagged :: Nil\n def dataType: DataType = tagged.dataType\n protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = tagged.genCode(ctx)\n protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)\n }\n\n /** Expression to tag columns from the right hand side of join expression. */\n case class DisambiguateRight[T](tagged: Expression) extends Expression with NonSQLExpression {\n def eval(input: InternalRow): Any = tagged.eval(input)\n def nullable: Boolean = false\n def children: Seq[Expression] = tagged :: Nil\n def dataType: DataType = tagged.dataType\n protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = tagged.genCode(ctx)\n protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)\n }\n}\n" }, { "path": "dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala", "content": "package org.apache.spark.sql\n\nimport org.apache.spark.sql.catalyst.ScalaReflection.{\n cleanUpReflectionObjects,\n getClassFromType,\n localTypeOf\n}\nimport org.apache.spark.sql.types.{\n BinaryType,\n BooleanType,\n ByteType,\n CalendarIntervalType,\n DataType,\n Decimal,\n DecimalType,\n DoubleType,\n FloatType,\n IntegerType,\n LongType,\n NullType,\n ObjectType,\n ShortType\n}\nimport org.apache.spark.unsafe.types.CalendarInterval\n\n/**\n * Copy of spark's pre 3.4 reflection based encoding\n */\npackage object reflection {\n\n /**\n * copy of pre 3.5.0 isNativeType, https://issues.apache.org/jira/browse/SPARK-44343 removed it\n */\n def isNativeType(dt: DataType): Boolean = dt match {\n case NullType | BooleanType | ByteType | ShortType | IntegerType |\n LongType | FloatType | DoubleType | BinaryType | CalendarIntervalType =>\n true\n case _ => false\n }\n\n private object ScalaSubtypeLock\n\n val universe: scala.reflect.runtime.universe.type =\n scala.reflect.runtime.universe\n\n import universe._\n\n /**\n * Returns the Spark SQL DataType for a given scala type. Where this is not an exact mapping\n * to a native type, an ObjectType is returned. Special handling is also used for Arrays including\n * those that hold primitive types.\n *\n * Unlike `schemaFor`, this function doesn't do any massaging of types into the Spark SQL type\n * system. As a result, ObjectType will be returned for things like boxed Integers\n */\n def dataTypeFor[T: TypeTag]: DataType = dataTypeFor(localTypeOf[T])\n\n /**\n * Synchronize to prevent concurrent usage of `<:<` operator.\n * This operator is not thread safe in any current version of scala; i.e.\n * (2.11.12, 2.12.10, 2.13.0-M5).\n *\n * See https://github.com/scala/bug/issues/10766\n */\n private[sql] def isSubtype(tpe1: `Type`, tpe2: `Type`): Boolean = {\n ScalaSubtypeLock.synchronized {\n tpe1 <:< tpe2\n }\n }\n\n private def dataTypeFor(tpe: `Type`): DataType = cleanUpReflectionObjects {\n tpe.dealias match {\n case t if isSubtype(t, definitions.NullTpe) => NullType\n case t if isSubtype(t, definitions.IntTpe) => IntegerType\n case t if isSubtype(t, definitions.LongTpe) => LongType\n case t if isSubtype(t, definitions.DoubleTpe) => DoubleType\n case t if isSubtype(t, definitions.FloatTpe) => FloatType\n case t if isSubtype(t, definitions.ShortTpe) => ShortType\n case t if isSubtype(t, definitions.ByteTpe) => ByteType\n case t if isSubtype(t, definitions.BooleanTpe) => BooleanType\n case t if isSubtype(t, localTypeOf[Array[Byte]]) => BinaryType\n case t if isSubtype(t, localTypeOf[CalendarInterval]) =>\n CalendarIntervalType\n case t if isSubtype(t, localTypeOf[Decimal]) => DecimalType.SYSTEM_DEFAULT\n case _ =>\n /* original Spark code checked for scala.Array vs ObjectType,\n this (and associated code) isn't needed due to TypedEncoders arrayEncoder */\n val clazz = getClassFromType(tpe)\n ObjectType(clazz)\n }\n }\n\n}\n" }, { "path": "dataset/src/main/spark-3/frameless/MapGroups.scala", "content": "package frameless\n\nimport org.apache.spark.sql.Encoder\nimport org.apache.spark.sql.catalyst.expressions.Attribute\nimport org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MapGroups => SMapGroups}\n\nobject MapGroups {\n def apply[K: Encoder, T: Encoder, U: Encoder](\n func: (K, Iterator[T]) => TraversableOnce[U],\n groupingAttributes: Seq[Attribute],\n dataAttributes: Seq[Attribute],\n child: LogicalPlan\n ): LogicalPlan = SMapGroups(func, groupingAttributes, dataAttributes, child)\n}\n" }, { "path": "dataset/src/main/spark-3.4+/frameless/MapGroups.scala", "content": "package frameless\n\nimport org.apache.spark.sql.Encoder\nimport org.apache.spark.sql.catalyst.expressions.Attribute\nimport org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, MapGroups => SMapGroups}\n\nobject MapGroups {\n def apply[K: Encoder, T: Encoder, U: Encoder](\n func: (K, Iterator[T]) => TraversableOnce[U],\n groupingAttributes: Seq[Attribute],\n dataAttributes: Seq[Attribute],\n child: LogicalPlan\n ): LogicalPlan =\n SMapGroups(\n func,\n groupingAttributes,\n dataAttributes,\n Seq(), // #698 - no order given\n child\n )\n}\n" }, { "path": "dataset/src/test/resources/log4j.properties", "content": "log4j.logger.akka.event.slf4j.Slf4jLogger=ERROR\nlog4j.logger.akka.event.slf4j=ERROR\nlog4j.logger.akka.remote.EndpointWriter=ERROR\nlog4j.logger.akka.remote.RemoteActorRefProvider$RemotingTerminator=ERROR\nlog4j.logger.com.anjuke.dm=ERROR\nlog4j.logger.io.netty.bootstrap.ServerBootstrap=ERROR\nlog4j.logger.io.netty.buffer.ByteBufUtil=ERROR\nlog4j.logger.io.netty.buffer.PooledByteBufAllocator=ERROR\nlog4j.logger.io.netty.channel.AbstractChannel=ERROR\nlog4j.logger.io.netty.channel.ChannelInitializer=ERROR\nlog4j.logger.io.netty.channel.ChannelOutboundBuffer=ERROR\nlog4j.logger.io.netty.channel.DefaultChannelPipeline=ERROR\nlog4j.logger.io.netty.channel.MultithreadEventLoopGroup=ERROR\nlog4j.logger.io.netty.channel.nio.AbstractNioChannel=ERROR\nlog4j.logger.io.netty.channel.nio.NioEventLoop=ERROR\nlog4j.logger.io.netty.channel.socket.nio.NioServerSocketChannel=ERROR\nlog4j.logger.io.netty.util.concurrent.DefaultPromise.rejectedExecution=ERROR\nlog4j.logger.io.netty.util.concurrent.DefaultPromise=ERROR\nlog4j.logger.io.netty.util.concurrent.GlobalEventExecutor=ERROR\nlog4j.logger.io.netty.util.concurrent.SingleThreadEventExecutor=ERROR\nlog4j.logger.io.netty.util.internal.logging.InternalLoggerFactory=ERROR\nlog4j.logger.io.netty.util.internal.PlatformDependent0=ERROR\nlog4j.logger.io.netty.util.internal.PlatformDependent=ERROR\nlog4j.logger.io.netty.util.internal.SystemPropertyUtil=ERROR\nlog4j.logger.io.netty.util.internal.ThreadLocalRandom=ERROR\nlog4j.logger.io.netty.util.NetUtil=ERROR\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=ERROR\nlog4j.logger.org.apache.hadoop.conf.Configuration=ERROR\nlog4j.logger.org.apache.hadoop.fs.FileSystem=ERROR\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=ERROR\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\nlog4j.logger.org.apache.hadoop.mapred.JobConf=ERROR\nlog4j.logger.org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.Interns=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.MetricsSourceBuilder=ERROR\nlog4j.logger.org.apache.hadoop.metrics2.lib.MutableMetricsFactory=ERROR\nlog4j.logger.org.apache.hadoop.security.authentication.util.KerberosName=ERROR\nlog4j.logger.org.apache.hadoop.security.Groups=ERROR\nlog4j.logger.org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback=ERROR\nlog4j.logger.org.apache.hadoop.security.SecurityUtil=ERROR\nlog4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR\nlog4j.logger.org.apache.hadoop.security.UserGroupInformation=ERROR\nlog4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR\nlog4j.logger.org.apache.hadoop.util.ShutdownHookManager=ERROR\nlog4j.logger.org.apache.spark.broadcast.TorrentBroadcast=ERROR\nlog4j.logger.org.apache.spark.ContextCleaner=ERROR\nlog4j.logger.org.apache.spark.executor.Executor=ERROR\nlog4j.logger.org.apache.spark.HeartbeatReceiver=ERROR\nlog4j.logger.org.apache.spark.HttpFileServer=ERROR\nlog4j.logger.org.apache.spark.HttpServer=ERROR\nlog4j.logger.org.apache.spark.MapOutputTrackerMaster=ERROR\nlog4j.logger.org.apache.spark.MapOutputTrackerMasterEndpoint=ERROR\nlog4j.logger.org.apache.spark.metrics.MetricsSystem=ERROR\nlog4j.logger.org.apache.spark.network.client.TransportClientFactory=ERROR\nlog4j.logger.org.apache.spark.network.netty.NettyBlockTransferService=ERROR\nlog4j.logger.org.apache.spark.network.protocol.MessageDecoder=ERROR\nlog4j.logger.org.apache.spark.network.protocol.MessageEncoder=ERROR\nlog4j.logger.org.apache.spark.network.server.OneForOneStreamManager=ERROR\nlog4j.logger.org.apache.spark.network.server.TransportServer=ERROR\nlog4j.logger.org.apache.spark.network.TransportContext=ERROR\nlog4j.logger.org.apache.spark.network.util.JavaUtils=ERROR\nlog4j.logger.org.apache.spark.rdd.CoGroupedRDD=ERROR\nlog4j.logger.org.apache.spark.rdd.SubtractedRDD=ERROR\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR\nlog4j.logger.org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1=ERROR\nlog4j.logger.org.apache.spark.scheduler.DAGScheduler=ERROR\nlog4j.logger.org.apache.spark.scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint=ERROR\nlog4j.logger.org.apache.spark.scheduler.TaskSchedulerImpl=ERROR\nlog4j.logger.org.apache.spark.scheduler.TaskSetManager=ERROR\nlog4j.logger.org.apache.spark.SecurityManager=ERROR\nlog4j.logger.org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter=ERROR\nlog4j.logger.org.apache.spark.SparkContext=ERROR\nlog4j.logger.org.apache.spark.SparkEnv=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GeneratePredicate=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeRowJoiner=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.package$ExpressionCanonicalizer=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.optimizer.DefaultOptimizer=ERROR\nlog4j.logger.org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys=ERROR\nlog4j.logger.org.apache.spark.sql.execution.aggregate.SortBasedAggregate=ERROR\nlog4j.logger.org.apache.spark.sql.execution.aggregate.TungstenAggregate=ERROR\nlog4j.logger.org.apache.spark.sql.execution.Exchange=ERROR\nlog4j.logger.org.apache.spark.sql.execution.joins.ShuffledHashOuterJoin=ERROR\nlog4j.logger.org.apache.spark.sql.SQLContext$$anon$1=ERROR\nlog4j.logger.org.apache.spark.sql.SQLContext$$anon$2=ERROR\nlog4j.logger.org.apache.spark.SSLOptions=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManager=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerInfo=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerMaster=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerMasterEndpoint=ERROR\nlog4j.logger.org.apache.spark.storage.BlockManagerSlaveEndpoint=ERROR\nlog4j.logger.org.apache.spark.storage.DiskBlockManager=ERROR\nlog4j.logger.org.apache.spark.storage.MemoryStore=ERROR\nlog4j.logger.org.apache.spark.storage.ShuffleBlockFetcherIterator=ERROR\nlog4j.logger.org.apache.spark.ui.SparkUI=ERROR\nlog4j.logger.org.apache.spark.unsafe.map.BytesToBytesMap=ERROR\nlog4j.logger.org.apache.spark.unsafe.memory.TaskMemoryManager=ERROR\nlog4j.logger.org.apache.spark.util.AkkaUtils=ERROR\nlog4j.logger.org.apache.spark.util.ClosureCleaner=ERROR\nlog4j.logger.org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter=ERROR\nlog4j.logger.org.apache.spark.util.Utils=ERROR\nlog4j.logger.org.apache.spark=ERROR\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.eclipse.jetty=ERROR\nlog4j.logger.org.spark-project.jetty.http.AbstractGenerator=ERROR\nlog4j.logger.org.spark-project.jetty.http.HttpGenerator=ERROR\nlog4j.logger.org.spark-project.jetty.http.MimeTypes=ERROR\nlog4j.logger.org.spark-project.jetty.io.AbstractBuffer=ERROR\nlog4j.logger.org.spark-project.jetty.io.nio=ERROR\nlog4j.logger.org.spark-project.jetty.server.AbstractConnector=ERROR\nlog4j.logger.org.spark-project.jetty.server.bio.SocketConnector=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.AbstractHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ContextHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ContextHandlerCollection=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.DefaultHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ErrorHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.GzipHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.handler.ResourceHandler=ERROR\nlog4j.logger.org.spark-project.jetty.server.Server=ERROR\nlog4j.logger.org.spark-project.jetty.server=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.DefaultServlet=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.Holder=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.ServletHandler=ERROR\nlog4j.logger.org.spark-project.jetty.servlet.ServletHolder=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.AggregateLifeCycle=ERROR\nlog4j.logger.org.spark-project.jetty.util.component.Container=ERROR\nlog4j.logger.org.spark-project.jetty.util.IO=ERROR\nlog4j.logger.org.spark-project.jetty.util.log=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.FileResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.JarFileResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.JarResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.Resource=ERROR\nlog4j.logger.org.spark-project.jetty.util.resource.URLResource=ERROR\nlog4j.logger.org.spark-project.jetty.util.StringUtil=ERROR\nlog4j.logger.org.spark-project.jetty.util.thread.QueuedThreadPool=ERROR\nlog4j.logger.org.spark-project.jetty.util.thread.Timeout=ERROR\nlog4j.logger.org.spark-project.jetty=ERROR\nlog4j.logger.Remoting=ERROR" }, { "path": "dataset/src/test/resources/log4j2.properties", "content": "# Set to debug or trace if log4j initialization is failing\nstatus = warn\n\n# Name of the configuration\nname = ConsoleAppender\n\n# Console appender configuration\nappender.console.type = Console\nappender.console.name = consoleLogger\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{YYYY-MM-dd HH:mm:ss} [%t] %-5p %c:%L - %m%n\nappender.console.target = SYSTEM_OUT\n\n# Root logger level\nrootLogger.level = error\n\n# Root logger referring to console appender\nrootLogger.appenderRef.stdout.ref = consoleLogger\n\nlogger.spark.name = org.apache.spark\nlogger.spark.level = warn\n\nlogger.hadoop.name = org.apache.hadoop\nlogger.hadoop.level = warn\n\n# To debug expressions:\n#logger.codegen.name = org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator\n#logger.codegen.level = debug" }, { "path": "dataset/src/test/scala/frameless/AsTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass AsTests extends TypedDatasetSuite {\n test(\"as[X2[A, B]]\") {\n def prop[A, B](data: Vector[(A, B)])(\n implicit\n eab: TypedEncoder[(A, B)],\n ex2: TypedEncoder[X2[A, B]]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n\n val dataset2 = dataset.as[X2[A,B]]().collect().run().toVector\n val data2 = data.map { case (a, b) => X2(a, b) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[String, String] _))\n check(forAll(prop[String, Int] _))\n check(forAll(prop[Long, Int] _))\n check(forAll(prop[Seq[Seq[Option[Seq[Long]]]], Seq[Int]] _))\n check(forAll(prop[Seq[Option[Seq[String]]], Seq[Int]] _))\n }\n\n test(\"as[X2[X2[A, B], C]\") {\n def prop[A, B, C](data: Vector[(A, B, C)])(\n implicit\n eab: TypedEncoder[((A, B), C)],\n ex2: TypedEncoder[X2[X2[A, B], C]]\n ): Prop = {\n val data2 = data.map {\n case (a, b, c) => ((a, b), c)\n }\n val dataset = TypedDataset.create(data2)\n\n val dataset2 = dataset.as[X2[X2[A,B], C]]().collect().run().toVector\n val data3 = data2.map { case ((a, b), c) => X2(X2(a, b), c) }\n\n dataset2 ?= data3\n }\n\n check(forAll(prop[String, Int, Int] _))\n check(forAll(prop[String, Int, String] _))\n check(forAll(prop[String, String, Int] _))\n check(forAll(prop[Long, Int, String] _))\n check(forAll(prop[Seq[Seq[Option[Seq[Long]]]], Seq[Int], Option[Seq[Option[Int]]]] _))\n check(forAll(prop[Seq[Option[Seq[String]]], Seq[Int], Seq[Option[String]]] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/BitwiseTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\n\nclass BitwiseTests extends TypedDatasetSuite with Matchers {\n\n /**\n * providing instances with implementations for bitwise operations since in the tests\n * we need to check the results from frameless vs the results from normal scala operators\n * for Numeric it is easy to test since scala comes with Numeric typeclass but there seems\n * to be no equivalent typeclass for bitwise ops for Byte Short Int and Long types supported in Catalyst\n */\n trait CatalystBitwise4Tests[A]{\n def bitwiseAnd(a1: A, a2: A): A\n def bitwiseOr(a1: A, a2: A): A\n def bitwiseXor(a1: A, a2: A): A\n def &(a1: A, a2: A): A = bitwiseAnd(a1, a2)\n def |(a1: A, a2: A): A = bitwiseOr(a1, a2)\n def ^(a1: A, a2: A): A = bitwiseXor(a1, a2)\n }\n\n object CatalystBitwise4Tests {\n implicit val framelessbyteBitwise : CatalystBitwise4Tests[Byte] = new CatalystBitwise4Tests[Byte] {\n def bitwiseOr(a1: Byte, a2: Byte) : Byte = (a1 | a2).toByte\n def bitwiseAnd(a1: Byte, a2: Byte): Byte = (a1 & a2).toByte\n def bitwiseXor(a1: Byte, a2: Byte): Byte = (a1 ^ a2).toByte\n }\n implicit val framelessshortBitwise : CatalystBitwise4Tests[Short] = new CatalystBitwise4Tests[Short] {\n def bitwiseOr(a1: Short, a2: Short) : Short = (a1 | a2).toShort\n def bitwiseAnd(a1: Short, a2: Short): Short = (a1 & a2).toShort\n def bitwiseXor(a1: Short, a2: Short): Short = (a1 ^ a2).toShort\n }\n implicit val framelessintBitwise : CatalystBitwise4Tests[Int] = new CatalystBitwise4Tests[Int] {\n def bitwiseOr(a1: Int, a2: Int) : Int = a1 | a2\n def bitwiseAnd(a1: Int, a2: Int): Int = a1 & a2\n def bitwiseXor(a1: Int, a2: Int): Int = a1 ^ a2\n }\n implicit val framelesslongBitwise : CatalystBitwise4Tests[Long] = new CatalystBitwise4Tests[Long] {\n def bitwiseOr(a1: Long, a2: Long) : Long = a1 | a2\n def bitwiseAnd(a1: Long, a2: Long): Long = a1 & a2\n def bitwiseXor(a1: Long, a2: Long): Long = a1 ^ a2\n }\n\n }\n import CatalystBitwise4Tests._\n test(\"bitwiseAND\") {\n def prop[A: TypedEncoder: CatalystBitwise](a: A, b: A)(\n implicit catalystBitwise4Tests: CatalystBitwise4Tests[A]\n ): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[CatalystBitwise4Tests[A]].bitwiseAnd(a, b)\n val resultSymbolic = implicitly[CatalystBitwise4Tests[A]].&(a, b)\n val got = df.select(df.col('a) bitwiseAND df.col('b)).collect().run()\n val gotSymbolic = df.select(df.col('a) & b).collect().run()\n val symbolicCol2Col = df.select(df.col('a) & df.col('b)).collect().run()\n val canCast = df.select(df.col('a).cast[Long] & 0L).collect().run()\n canCast should contain theSameElementsAs Seq.fill[Long](gotSymbolic.size)(0L)\n result ?= resultSymbolic\n symbolicCol2Col ?= (result :: Nil)\n got ?= (result :: Nil)\n gotSymbolic ?= (resultSymbolic :: Nil)\n }\n\n check(prop[Byte] _)\n check(prop[Short] _)\n check(prop[Int] _)\n check(prop[Long] _)\n }\n\n test(\"bitwiseOR\") {\n def prop[A: TypedEncoder: CatalystBitwise](a: A, b: A)(\n implicit catalystBitwise4Tests: CatalystBitwise4Tests[A]\n ): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[CatalystBitwise4Tests[A]].bitwiseOr(a, b)\n val resultSymbolic = implicitly[CatalystBitwise4Tests[A]].|(a, b)\n val got = df.select(df.col('a) bitwiseOR df.col('b)).collect().run()\n val gotSymbolic = df.select(df.col('a) | b).collect().run()\n val symbolicCol2Col = df.select(df.col('a) | df.col('b)).collect().run()\n val canCast = df.select(df.col('a).cast[Long] | -1L).collect().run()\n canCast should contain theSameElementsAs Seq.fill[Long](gotSymbolic.size)(-1L)\n result ?= resultSymbolic\n symbolicCol2Col ?= (result :: Nil)\n got ?= (result :: Nil)\n gotSymbolic ?= (resultSymbolic :: Nil)\n }\n\n check(prop[Byte] _)\n check(prop[Short] _)\n check(prop[Int] _)\n check(prop[Long] _)\n }\n\n test(\"bitwiseXOR\") {\n def prop[A: TypedEncoder: CatalystBitwise](a: A, b: A)(\n implicit catalystBitwise4Tests: CatalystBitwise4Tests[A]\n ): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[CatalystBitwise4Tests[A]].bitwiseXor(a, b)\n val resultSymbolic = implicitly[CatalystBitwise4Tests[A]].^(a, b)\n result ?= resultSymbolic\n val got = df.select(df.col('a) bitwiseXOR df.col('b)).collect().run()\n val gotSymbolic = df.select(df.col('a) ^ b).collect().run()\n val zeroes = df.select(df.col('a) ^ df.col('a)).collect().run()\n zeroes should contain theSameElementsAs Seq.fill[Long](gotSymbolic.size)(0L)\n got ?= (result :: Nil)\n gotSymbolic ?= (resultSymbolic :: Nil)\n }\n\n check(prop[Byte] _)\n check(prop[Short] _)\n check(prop[Int] _)\n check(prop[Long] _)\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/CastTests.scala", "content": "package frameless\n\nimport org.scalacheck.{Arbitrary, Gen, Prop}\nimport org.scalacheck.Prop._\n\nclass CastTests extends TypedDatasetSuite {\n\n def prop[A: TypedEncoder, B: TypedEncoder](f: A => B)(a: A)(\n implicit\n cast: CatalystCast[A, B]\n ): Prop = {\n val df = TypedDataset.create(X1(a) :: Nil)\n val got = df.select(df.col('a).cast[B]).collect().run()\n\n got ?= (f(a) :: Nil)\n }\n\n test(\"cast\") {\n // numericToDecimal\n check(prop[BigDecimal, BigDecimal](identity) _)\n check(prop[Byte, BigDecimal](x => BigDecimal.valueOf(x.toLong)) _)\n check(prop[Double, BigDecimal](BigDecimal.valueOf) _)\n check(prop[Int, BigDecimal](x => BigDecimal.valueOf(x.toLong)) _)\n check(prop[Long, BigDecimal](BigDecimal.valueOf) _)\n check(prop[Short, BigDecimal](x => BigDecimal.valueOf(x.toLong)) _)\n\n // numericToByte\n check(prop[BigDecimal, Byte](_.toByte) _)\n check(prop[Byte, Byte](identity) _)\n check(prop[Double, Byte](_.toByte) _)\n check(prop[Int, Byte](_.toByte) _)\n check(prop[Long, Byte](_.toByte) _)\n check(prop[Short, Byte](_.toByte) _)\n\n // numericToDouble\n check(prop[BigDecimal, Double](_.toDouble) _)\n check(prop[Byte, Double](_.toDouble) _)\n check(prop[Double, Double](identity) _)\n check(prop[Int, Double](_.toDouble) _)\n check(prop[Long, Double](_.toDouble) _)\n check(prop[Short, Double](_.toDouble) _)\n\n // numericToInt\n check(prop[BigDecimal, Int](_.toInt) _)\n check(prop[Byte, Int](_.toInt) _)\n check(prop[Double, Int](_.toInt) _)\n check(prop[Int, Int](identity) _)\n check(prop[Long, Int](_.toInt) _)\n check(prop[Short, Int](_.toInt) _)\n\n // numericToLong\n check(prop[BigDecimal, Long](_.toLong) _)\n check(prop[Byte, Long](_.toLong) _)\n check(prop[Double, Long](_.toLong) _)\n check(prop[Int, Long](_.toLong) _)\n check(prop[Long, Long](identity) _)\n check(prop[Short, Long](_.toLong) _)\n\n // numericToShort\n check(prop[BigDecimal, Short](_.toShort) _)\n check(prop[Byte, Short](_.toShort) _)\n check(prop[Double, Short](_.toShort) _)\n check(prop[Int, Short](_.toShort) _)\n check(prop[Long, Short](_.toShort) _)\n check(prop[Short, Short](identity) _)\n\n // castToString\n // TODO compare without trailing zeros\n // check(prop[BigDecimal, String](_.toString()) _)\n check(prop[Byte, String](_.toString) _)\n check(prop[Double, String](_.toString) _)\n check(prop[Int, String](_.toString) _)\n check(prop[Long, String](_.toString) _)\n check(prop[Short, String](_.toString) _)\n\n // stringToBoolean\n val trueStrings = Set(\"t\", \"true\", \"y\", \"yes\", \"1\")\n val falseStrings = Set(\"f\", \"false\", \"n\", \"no\", \"0\")\n\n def stringToBoolean(str: String): Option[Boolean] = {\n if (trueStrings(str)) Some(true)\n else if (falseStrings(str)) Some(false)\n else None\n }\n\n val stringToBooleanGen = Gen.oneOf(\n Gen.oneOf(trueStrings.toSeq),\n Gen.oneOf(falseStrings.toSeq),\n Arbitrary.arbitrary[String]\n )\n\n check(forAll(stringToBooleanGen)(prop(stringToBoolean)))\n\n // xxxToBoolean\n check(prop[BigDecimal, Boolean](_ != BigDecimal(0)) _)\n check(prop[Byte, Boolean](_ != 0) _)\n check(prop[Double, Boolean](_ != 0) _)\n check(prop[Int, Boolean](_ != 0) _)\n check(prop[Long, Boolean](_ != 0L) _)\n check(prop[Short, Boolean](_ != 0) _)\n\n // booleanToNumeric\n check(prop[Boolean, BigDecimal](x => if (x) BigDecimal(1) else BigDecimal(0)) _)\n check(prop[Boolean, Byte](x => if (x) 1 else 0) _)\n check(prop[Boolean, Double](x => if (x) 1.0f else 0.0f) _)\n check(prop[Boolean, Int](x => if (x) 1 else 0) _)\n check(prop[Boolean, Long](x => if (x) 1L else 0L) _)\n check(prop[Boolean, Short](x => if (x) 1 else 0) _)\n }\n\n}\n" }, { "path": "dataset/src/test/scala/frameless/ColTests.scala", "content": "package frameless\n\nimport shapeless.test.illTyped\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass ColTests extends TypedDatasetSuite {\n test(\"col\") {\n val x4 = TypedDataset.create[X4[Int, String, Long, Boolean]](Nil)\n val t4 = TypedDataset.create[(Int, String, Long, Boolean)](Nil)\n\n x4.col('a)\n t4.col('_1)\n\n x4.col[Int]('a)\n t4.col[Int]('_1)\n\n illTyped(\"x4.col[String]('a)\", \"No column .* of type String in frameless.X4.*\")\n\n x4.col('b)\n t4.col('_2)\n\n x4.col[String]('b)\n t4.col[String]('_2)\n\n illTyped(\"x4.col[Int]('b)\", \"No column .* of type Int in frameless.X4.*\")\n\n ()\n }\n\n test(\"colMany\") {\n type X2X2 = X2[X2[Int, String], X2[Long, Boolean]]\n val x2x2 = TypedDataset.create[X2X2](Nil)\n\n val aa: TypedColumn[X2X2, Int] = x2x2.colMany('a, 'a)\n val ab: TypedColumn[X2X2, String] = x2x2.colMany('a, 'b)\n val ba: TypedColumn[X2X2, Long] = x2x2.colMany('b, 'a)\n val bb: TypedColumn[X2X2, Boolean] = x2x2.colMany('b, 'b)\n\n illTyped(\"x2x2.colMany('a, 'c)\")\n illTyped(\"x2x2.colMany('a, 'a, 'a)\")\n }\n\n test(\"select colMany\") {\n def prop[A: TypedEncoder](x: X2[X2[A, A], A]): Prop = {\n val df = TypedDataset.create(x :: Nil)\n val got = df.select(df.colMany('a, 'a)).collect().run()\n\n got ?= (x.a.a :: Nil)\n }\n\n check(prop[Int] _)\n check(prop[X2[Int, Int]] _)\n check(prop[X2[X2[Int, Int], Int]] _)\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/CollectTests.scala", "content": "package frameless\n\nimport frameless.CollectTests.{ prop, propArray }\nimport org.apache.spark.sql.SparkSession\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport scala.reflect.ClassTag\n\nclass CollectTests extends TypedDatasetSuite {\n test(\"collect()\") {\n check(forAll(propArray[Int] _))\n check(forAll(propArray[Long] _))\n check(forAll(propArray[Boolean] _))\n check(forAll(propArray[Float] _))\n check(forAll(propArray[String] _))\n check(forAll(propArray[Byte] _))\n check(forAll(propArray[Option[Int]] _))\n check(forAll(propArray[Option[Long]] _))\n check(forAll(propArray[Option[Double]] _))\n check(forAll(propArray[Option[Float]] _))\n check(forAll(propArray[Option[Short]] _))\n check(forAll(propArray[Option[Byte]] _))\n check(forAll(propArray[Option[Boolean]] _))\n check(forAll(propArray[Option[String]] _))\n\n check(forAll(prop[X2[Int, Int]] _))\n check(forAll(prop[X2[String, String]] _))\n check(forAll(prop[X2[String, Int]] _))\n check(forAll(prop[X2[Long, Int]] _))\n\n check(forAll(prop[X2[X2[Int, String], Boolean]] _))\n check(forAll(prop[Tuple1[Option[Int]]] _))\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Char] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[String] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[Option[Int]] _))\n check(forAll(prop[Option[Long]] _))\n check(forAll(prop[Option[Double]] _))\n check(forAll(prop[Option[Float]] _))\n check(forAll(prop[Option[Short]] _))\n check(forAll(prop[Option[Byte]] _))\n check(forAll(prop[Option[Boolean]] _))\n check(forAll(prop[Option[String]] _))\n check(forAll(prop[Option[SQLDate]] _))\n check(forAll(prop[Option[SQLTimestamp]] _))\n\n check(forAll(prop[Vector[Int]] _))\n check(forAll(prop[List[Int]] _))\n check(forAll(prop[Seq[Int]] _))\n check(forAll(prop[Vector[Char]] _))\n check(forAll(prop[List[Char]] _))\n check(forAll(prop[Seq[Char]] _))\n check(forAll(prop[Set[Char]] _))\n check(forAll(prop[Seq[Seq[Seq[Char]]]] _))\n check(forAll(prop[Seq[Option[String]]] _))\n check(forAll(prop[Seq[Map[String, Long]]] _))\n check(forAll(prop[Seq[Map[String, X2[Option[Long], Vector[Boolean]]]]] _))\n check(forAll(prop[Option[Int]] _))\n check(forAll(prop[Vector[X2[Int, Int]]] _))\n\n check(forAll(prop[X1[Vector[Food]]] _))\n check(forAll(prop[X1[Vector[X1[Food]]]] _))\n check(forAll(prop[X1[Vector[X1[Int]]]] _))\n\n // TODO this doesn't work, and never worked...\n // check(forAll(prop[X1[Option[X1[Option[Int]]]]] _))\n\n check(forAll(prop[UdtEncodedClass] _))\n check(forAll(prop[Option[UdtEncodedClass]] _))\n check(forAll(prop[X1[UdtEncodedClass]] _))\n check(forAll(prop[X2[Int, UdtEncodedClass]] _))\n check(forAll(prop[(Long, UdtEncodedClass)] _))\n }\n}\n\nobject CollectTests {\n import frameless.syntax._\n\n def prop[A: TypedEncoder : ClassTag](data: Vector[A])(implicit c: SparkSession): Prop =\n TypedDataset.create(data).collect().run().toVector ?= data\n\n def propArray[A: TypedEncoder : ClassTag](data: Vector[X1[Array[A]]])(implicit c: SparkSession): Prop =\n Prop(TypedDataset.create(data).collect().run().toVector.zip(data).forall {\n case (X1(l), X1(r)) => l.sameElements(r)\n })\n}\n" }, { "path": "dataset/src/test/scala/frameless/ColumnTests.scala", "content": "package frameless\n\nimport java.util.Date\nimport java.math.BigInteger\n\nimport java.time.{ Instant, LocalDate, Period, Duration }\nimport java.time.temporal.ChronoUnit\n\nimport java.sql.{ Date => SqlDate, Timestamp }\n\nimport scala.math.Ordering.Implicits._\nimport scala.util.Try\n\nimport org.scalacheck.{ Arbitrary, Gen, Prop }, Arbitrary.arbitrary, Prop._\n\nimport org.scalatest.matchers.should.Matchers\n\nimport shapeless.test.illTyped\n\nfinal class ColumnTests extends TypedDatasetSuite with Matchers {\n\n implicit val timestampArb: Arbitrary[Timestamp] = Arbitrary {\n OrderingImplicits.arbInstant.arbitrary.map { i =>\n Timestamp from i.truncatedTo(ChronoUnit.MILLIS)\n }\n }\n\n implicit val dateArb: Arbitrary[Date] = Arbitrary {\n OrderingImplicits.arbInstant.arbitrary.map(Date from _)\n }\n\n private implicit object OrderingImplicits {\n implicit val sqlDateOrdering: Ordering[SQLDate] = Ordering.by(_.days)\n\n implicit val sqlTimestmapOrdering: Ordering[SQLTimestamp] =\n Ordering.by(_.us)\n\n implicit val periodOrdering: Ordering[Period] =\n Ordering.by(p => (p.getYears, p.getMonths, p.getDays))\n\n /**\n * DateTimeUtils.instantToMicros supports dates starting 1970-01-01T00:00:00Z, which is Instant.EPOCH.\n * This function also overflows on Instant.MAX, to be sure it never overflows we use Instant.MAX / 4.\n * For implementation details check the org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros function details.\n */\n val genInstant = Gen.choose[Instant](\n Instant.EPOCH,\n Instant.ofEpochMilli(Instant.MAX.getEpochSecond / 4)\n )\n implicit val arbInstant: Arbitrary[Instant] = Arbitrary(genInstant)\n\n implicit val arbDuration: Arbitrary[Duration] = Arbitrary(\n genInstant.map(i => Duration.ofMillis(i.toEpochMilli))\n )\n\n implicit val arbPeriod: Arbitrary[Period] = Arbitrary(\n Gen.chooseNum(0, Int.MaxValue).map(l => Period.of(l, l, l))\n )\n }\n\n test(\"select('a < 'b, 'a <= 'b, 'a > 'b, 'a >= 'b)\") {\n import OrderingImplicits._\n\n def prop[A: TypedEncoder: CatalystOrdered: Ordering](a: A, b: A): Prop = {\n val dataset = TypedDataset.create(X2(a, b) :: Nil)\n val A = dataset.col('a)\n val B = dataset.col('b)\n\n val dataset2 = dataset\n .selectMany(\n A < B,\n A < b, // One test uses columns, other uses literals\n A <= B,\n A <= b,\n A > B,\n A > b,\n A >= B,\n A >= b\n )\n .collect()\n .run()\n .toVector\n\n dataset2 ?= Vector(\n (a < b, a < b, a <= b, a <= b, a > b, a > b, a >= b, a >= b)\n )\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Instant] _))\n check(forAll(prop[Duration] _))\n check(forAll(prop[Period] _))\n }\n\n test(\"between\") {\n import OrderingImplicits._\n def prop[A: TypedEncoder: CatalystOrdered: Ordering](\n a: A,\n b: A,\n c: A\n ): Prop = {\n val dataset = TypedDataset.create(X3(a, b, c) :: Nil)\n val A = dataset.col('a)\n val B = dataset.col('b)\n val C = dataset.col('c)\n\n val isBetweeen = dataset\n .selectMany(A.between(B, C), A.between(b, c))\n .collect()\n .run()\n .toVector\n val result = b <= a && a <= c\n\n isBetweeen ?= Vector((result, result))\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Instant] _))\n check(forAll(prop[Duration] _))\n check(forAll(prop[Period] _))\n }\n\n test(\"toString\") {\n val t = TypedDataset.create((1, 2) :: Nil)\n t('_1).toString ?= t.dataset.col(\"_1\").toString()\n }\n\n test(\"boolean and / or\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (s: Seq[X3[Boolean, Boolean, Boolean]]) =>\n val ds = TypedDataset.create(s)\n\n val typedBoolean = ds\n .select(\n ds('a) && ds('b) || ds('c),\n ds('a).and(ds('b)).or(ds('c))\n )\n .collect()\n .run()\n .toList\n\n val untypedDs = ds.toDF()\n val untypedBoolean = untypedDs\n .select(\n untypedDs(\"a\") && untypedDs(\"b\") || untypedDs(\"c\"),\n untypedDs(\"a\").and(untypedDs(\"b\")).or(untypedDs(\"c\"))\n )\n .as[(Boolean, Boolean)]\n .collect()\n .toList\n\n typedBoolean ?= untypedBoolean\n }\n }\n }\n\n test(\"substr\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (a: String, b: Int, c: Int) =>\n val ds = TypedDataset.create(X3(a, b, c) :: Nil)\n\n val typedSubstr =\n ds.select(ds('a).substr(ds('b), ds('c))).collect().run().toList\n\n val untypedDs = ds.toDF()\n val untypedSubstr = untypedDs\n .select(untypedDs(\"a\").substr(untypedDs(\"b\"), untypedDs(\"c\")))\n .as[String]\n .collect()\n .toList\n\n typedSubstr ?= untypedSubstr\n }\n }\n\n check {\n forAll { (a: String, b: Int, c: Int) =>\n val ds = TypedDataset.create(X1(a) :: Nil)\n\n val typedSubstr = ds.select(ds('a).substr(b, c)).collect().run().toList\n\n val untypedDs = ds.toDF()\n val untypedSubstr = untypedDs\n .select(untypedDs(\"a\").substr(b, c))\n .as[String]\n .collect()\n .toList\n\n typedSubstr ?= untypedSubstr\n }\n }\n\n val ds1 = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds1.select(ds1('_1).substr(0, 5))\"\"\")\n illTyped(\"\"\"ds1.select(ds1('_2).substr(0, 5))\"\"\")\n illTyped(\"\"\"ds1.select(ds1('_3).substr(0, 5))\"\"\")\n illTyped(\"\"\"ds1.select(ds1('_1).substr(ds1('_2), ds1('_3)))\"\"\")\n }\n\n test(\"like\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (a: String, b: String) =>\n val ds = TypedDataset.create(X2(a, b) :: Nil)\n\n val typedLike =\n ds.select(ds('a).like(a), ds('b).like(a)).collect().run().toList\n\n val untypedDs = ds.toDF()\n val untypedLike = untypedDs\n .select(untypedDs(\"a\").like(a), untypedDs(\"b\").like(a))\n .as[(Boolean, Boolean)]\n .collect()\n .toList\n\n typedLike ?= untypedLike\n }\n }\n\n val ds = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds.select(ds('_1).like(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_2).like(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_3).like(\"foo\"))\"\"\")\n }\n\n test(\"rlike\") {\n val spark = session\n import spark.implicits._\n\n val regex = Gen.nonEmptyListOf(arbitrary[Char]).map(_.mkString).suchThat {\n str => Try(str.r).isSuccess\n }\n\n check {\n forAll(regex, arbitrary[String]) { (a, b) =>\n val ds = TypedDataset.create(X2(a, b) :: Nil)\n\n val typedLike = ds\n .select(ds('a).rlike(a), ds('b).rlike(a), ds('a).rlike(\".*\"))\n .collect()\n .run()\n .toList\n\n val untypedDs = ds.toDF()\n val untypedLike = untypedDs\n .select(\n untypedDs(\"a\").rlike(a),\n untypedDs(\"b\").rlike(a),\n untypedDs(\"a\").rlike(\".*\")\n )\n .as[(Boolean, Boolean, Boolean)]\n .collect()\n .toList\n\n typedLike ?= untypedLike\n }\n }\n\n val ds = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds.select(ds('_1).rlike(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_2).rlike(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_3).rlike(\"foo\"))\"\"\")\n }\n\n test(\"contains\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (a: String, b: String) =>\n val ds = TypedDataset.create(X2(a, b) :: Nil)\n\n val typedContains = ds\n .select(ds('a).contains(ds('b)), ds('b).contains(a))\n .collect()\n .run()\n .toList\n\n val untypedDs = ds.toDF()\n val untypedContains = untypedDs\n .select(\n untypedDs(\"a\").contains(untypedDs(\"b\")),\n untypedDs(\"b\").contains(a)\n )\n .as[(Boolean, Boolean)]\n .collect()\n .toList\n\n typedContains ?= untypedContains\n }\n }\n\n val ds = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds.select(ds('_1).contains(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_2).contains(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_3).contains(\"foo\"))\"\"\")\n }\n\n test(\"startsWith\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (a: String, b: String) =>\n val ds = TypedDataset.create(X2(a, b) :: Nil)\n\n val typedStartsWith = ds\n .select(ds('a).startsWith(ds('b)), ds('b).startsWith(a))\n .collect()\n .run()\n .toList\n\n val untypedDs = ds.toDF()\n val untypedStartsWith = untypedDs\n .select(\n untypedDs(\"a\").startsWith(untypedDs(\"b\")),\n untypedDs(\"b\").startsWith(a)\n )\n .as[(Boolean, Boolean)]\n .collect()\n .toList\n\n typedStartsWith ?= untypedStartsWith\n }\n }\n\n val ds = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds.select(ds('_1).startsWith(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_2).startsWith(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_3).startsWith(\"foo\"))\"\"\")\n }\n\n test(\"endsWith\") {\n val spark = session\n import spark.implicits._\n\n check {\n forAll { (a: String, b: String) =>\n val ds = TypedDataset.create(X2(a, b) :: Nil)\n val typedStartsWith = ds\n .select(ds('a).endsWith(ds('b)), ds('b).endsWith(a))\n .collect()\n .run()\n .toList\n\n val untypedDs = ds.toDF()\n val untypedStartsWith = untypedDs\n .select(\n untypedDs(\"a\").endsWith(untypedDs(\"b\")),\n untypedDs(\"b\").endsWith(a)\n )\n .as[(Boolean, Boolean)]\n .collect()\n .toList\n\n typedStartsWith ?= untypedStartsWith\n }\n }\n\n val ds = TypedDataset.create((1, false, 2.0) :: Nil)\n illTyped(\"\"\"ds.select(ds('_1).endsWith(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_2).endsWith(\"foo\"))\"\"\")\n illTyped(\"\"\"ds.select(ds('_3).endsWith(\"foo\"))\"\"\")\n }\n\n test(\"getOrElse\") {\n def prop[A: TypedEncoder](a: A, opt: Option[A]) = {\n val dataset = TypedDataset.create(X2(a, opt) :: Nil)\n\n val defaulted: (A, A) = dataset\n .select(dataset('b).getOrElse(dataset('a)), dataset('b).getOrElse(a))\n .collect()\n .run()\n .toList\n .head\n\n defaulted ?= (opt.getOrElse(a) -> opt.getOrElse(a))\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[Date] _))\n check(forAll(prop[Timestamp] _))\n check(forAll(prop[String] _))\n\n // Scalacheck is too slow\n check(prop[BigInt](BigInt(Long.MaxValue).+(BigInt(Long.MaxValue)), None))\n check(prop[BigInt](BigInt(\"0\"), Some(BigInt(Long.MaxValue))))\n check(\n prop[BigInt](\n BigInt(Long.MinValue).-(BigInt(Long.MinValue)),\n Some(BigInt(\"0\"))\n )\n )\n\n check(\n prop[BigInteger](\n BigInteger\n .valueOf(Long.MaxValue)\n .add(BigInteger.valueOf(Long.MaxValue)),\n None\n )\n )\n\n check(\n prop[BigInteger](\n BigInteger.valueOf(0L),\n Some(BigInteger.valueOf(Long.MaxValue))\n )\n )\n\n check(\n prop[BigInteger](\n BigInteger\n .valueOf(Long.MinValue)\n .subtract(BigInteger.valueOf(Long.MinValue)),\n Some(BigInteger.valueOf(0L))\n )\n )\n }\n\n test(\"Consistency with Spark internal date/time representation\") {\n val ts = Timestamp.from(Instant parse \"1990-01-01T01:00:00.000Z\")\n val date = Date.from(Instant parse \"1991-01-01T02:00:00.000Z\")\n\n val sqlDate = SqlDate.valueOf(LocalDate parse \"1991-02-01\")\n\n val input = Seq(X3(ts, date, sqlDate))\n\n val ds: TypedDataset[X3[Timestamp, Date, SqlDate]] =\n TypedDataset.create(input)\n\n val result1: Seq[(Timestamp, Date, SqlDate)] =\n ds.dataset.toDF\n .collect()\n .map { row =>\n Tuple3(\n row.getTimestamp(0),\n Date.from(row.getTimestamp(1).toInstant),\n row.getDate(2)\n )\n }\n .toSeq\n\n result1 shouldEqual Seq(Tuple3(ts, date, sqlDate))\n\n val result2: Seq[X3[Timestamp, Date, SqlDate]] =\n ds.collect.run().toSeq\n\n result2 shouldEqual input\n }\n\n test(\"asCol\") {\n def prop[A: TypedEncoder, B: TypedEncoder](a: Seq[X2[A, B]]) = {\n val ds: TypedDataset[X2[A, B]] = TypedDataset.create(a)\n\n val frameless: Seq[(A, X2[A, B], X2[A, B], X2[A, B], B)] =\n ds.select(ds('a), ds.asCol, ds.asCol, ds.asCol, ds('b)).collect().run()\n\n val scala: Seq[(A, X2[A, B], X2[A, B], X2[A, B], B)] =\n a.map(x => (x.a, x, x, x, x.b))\n\n scala ?= frameless\n }\n\n check(forAll(prop[Int, Option[Long]] _))\n check(forAll(prop[Vector[Char], Option[Boolean]] _))\n check(forAll(prop[Vector[Vector[String]], Vector[Vector[BigDecimal]]] _))\n }\n\n test(\"asCol single column TypedDatasets\") {\n def prop[A: TypedEncoder](a: Seq[A]) = {\n val ds: TypedDataset[A] = TypedDataset.create(a)\n\n val frameless: Seq[(A, A, A)] =\n ds.select(ds.asCol, ds.asCol, ds.asCol).collect().run()\n\n val scala: Seq[(A, A, A)] =\n a.map(x => (x, x, x))\n\n scala ?= frameless\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Date] _))\n check(forAll(prop[Vector[Vector[String]]] _))\n }\n\n test(\"asCol with numeric operators\") {\n def prop(a: Seq[Long]) = {\n val ds: TypedDataset[Long] = TypedDataset.create(a)\n val (first, second) = (2L, 5L)\n val frameless: Seq[(Long, Long, Long)] =\n ds.select(ds.asCol, ds.asCol + first, ds.asCol * second).collect().run()\n\n val scala: Seq[(Long, Long, Long)] =\n a.map(x => (x, x + first, x * second))\n\n scala ?= frameless\n }\n\n check(forAll(prop _))\n }\n\n test(\"reference Value class so can join on\") {\n import RecordEncoderTests.{ Name, Person }\n\n val bar = new Name(\"bar\")\n\n val ds1: TypedDataset[Person] =\n TypedDataset.create(Seq(Person(bar, 23), Person(new Name(\"foo\"), 11)))\n\n val ds2: TypedDataset[Name] =\n TypedDataset.create(Seq(new Name(\"lorem\"), bar))\n\n val joined = ds1.joinLeftSemi(ds2)(ds1.col('name) === ds2.asJoinColValue)\n\n joined.collect().run() shouldEqual Seq(Person(bar, 23))\n }\n\n test(\"unary_!\") {\n val ds = TypedDataset.create((true, false) :: Nil)\n\n val rs = ds.select(!ds('_1), !ds('_2)).collect().run().head\n\n rs shouldEqual (false -> true)\n }\n\n test(\"unary_! with non-boolean columns should not compile\") {\n val ds = TypedDataset.create((1, \"a\", 2.0) :: Nil)\n\n \"ds.select(!ds('_1))\" shouldNot typeCheck\n \"ds.select(!ds('_2))\" shouldNot typeCheck\n \"ds.select(!ds('_3))\" shouldNot typeCheck\n }\n\n test(\"opt\") {\n val data = (Option(1L), Option(2L)) :: (None, None) :: Nil\n val ds = TypedDataset.create(data)\n val rs =\n ds.select(ds('_1).opt.map(_ * 2), ds('_1).opt.map(_ + 2)).collect().run()\n val expected = data.map { case (x, y) => (x.map(_ * 2), y.map(_ + 1)) }\n rs shouldEqual expected\n }\n\n test(\"opt compiles only for columns of type Option[_]\") {\n val ds = TypedDataset.create((1, List(1, 2, 3)) :: Nil)\n \"ds.select(ds('_1).opt.map(x => x))\" shouldNot typeCheck\n \"ds.select(ds('_2).opt.map(x => x))\" shouldNot typeCheck\n }\n\n test(\"field\") {\n val ds = TypedDataset.create((1, (2.3F, \"a\")) :: Nil)\n val rs = ds.select(ds('_2).field('_2)).collect().run()\n\n rs shouldEqual Seq(\"a\")\n }\n\n test(\"field compiles only for valid field\") {\n val ds = TypedDataset.create((1, (2.3F, \"a\")) :: Nil)\n\n \"ds.select(ds('_2).field('_3))\" shouldNot typeCheck\n }\n\n test(\"col through lambda\") {\n case class MyClass1(a: Int, b: String, c: MyClass2)\n case class MyClass2(d: Long)\n\n val ds = TypedDataset.create(\n Seq(MyClass1(1, \"2\", MyClass2(3L)), MyClass1(4, \"5\", MyClass2(6L)))\n )\n\n assert(ds.col(_.a).isInstanceOf[TypedColumn[MyClass1, Int]])\n assert(ds.col(_.b).isInstanceOf[TypedColumn[MyClass1, String]])\n assert(ds.col(_.c.d).isInstanceOf[TypedColumn[MyClass1, Long]])\n\n \"ds.col(_.c.toString)\" shouldNot typeCheck\n \"ds.col(_.c.toInt)\" shouldNot typeCheck\n \"ds.col(x => java.lang.Math.abs(x.a))\" shouldNot typeCheck\n\n // we should be able to block the following as well...\n \"ds.col(_.a.toInt)\" shouldNot typeCheck\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ColumnViaLambdaTests.scala", "content": "package frameless\n\nimport org.scalatest.matchers.should.Matchers\nimport shapeless.test.illTyped\n\ncase class MyClass1(a: Int, b: String, c: MyClass2, g: Option[MyClass4])\ncase class MyClass2(d: Long, e: MyClass3)\ncase class MyClass3(f: Double)\ncase class MyClass4(h: Boolean)\n\nfinal class ColumnViaLambdaTests extends TypedDatasetSuite with Matchers {\n\n def ds = {\n TypedDataset.create(Seq(\n MyClass1(1, \"2\", MyClass2(3L, MyClass3(7.0D)), Some(MyClass4(true))),\n MyClass1(4, \"5\", MyClass2(6L, MyClass3(8.0D)), None)))\n }\n\n test(\"col(_.a)\") {\n val col = TypedColumn[MyClass1, Int](_.a)\n\n ds.select(col).collect().run() shouldEqual Seq(1, 4)\n }\n\n test(\"col(x => x.a\") {\n val col = TypedColumn[MyClass1, Int](x => x.a)\n\n ds.select(col).collect().run() shouldEqual Seq(1, 4)\n }\n\n test(\"col((x: MyClass1) => x.a\") {\n val col = TypedColumn { (x: MyClass1) => x.a }\n\n ds.select(col).collect().run() shouldEqual Seq(1, 4)\n }\n\n test(\"col((x: MyClass1) => x.c.e.f\") {\n val col = TypedColumn { (x: MyClass1) => x.c.e.f }\n\n ds.select(col).collect().run() shouldEqual Seq(7.0D, 8.0D)\n }\n\n test(\"col(_.c.d)\") {\n val col = TypedColumn[MyClass1, Long](_.c.d)\n\n ds.select(col).collect().run() shouldEqual Seq(3L, 6L)\n }\n\n test(\"col(_.c.e.f)\") {\n val col = TypedColumn[MyClass1, Double](_.c.e.f)\n\n ds.select(col).collect().run() shouldEqual Seq(7.0D, 8.0D)\n }\n\n test(\"col(_.c.d) as int does not compile (is long)\") {\n illTyped(\"TypedColumn[MyClass1, Int](_.c.d)\")\n }\n\n test(\"col(_.g.h does not compile\") {\n val col = ds.col(_.g) // the path \"ends\" at .g (can't access h)\n illTyped(\"\"\"ds.col(_.g.h)\"\"\")\n }\n\n test(\"col(_.a.toString) does not compile\") {\n illTyped(\"\"\"ds.col(_.a.toString)\"\"\")\n }\n\n test(\"col(_.a.toString.size) does not compile\") {\n illTyped(\"\"\"ds.col(_.a.toString.size)\"\"\")\n }\n\n test(\"col((x: MyClass1) => x.toString.size) does not compile\") {\n illTyped(\"\"\"ds.col((x: MyClass1) => x.toString.size)\"\"\")\n }\n\n test(\"col(x => java.lang.Math.abs(x.a)) does not compile\") {\n illTyped(\"\"\"col(x => java.lang.Math.abs(x.a))\"\"\")\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/CreateTests.scala", "content": "package frameless\n\nimport org.scalacheck.{Arbitrary, Prop}\nimport org.scalacheck.Prop._\n\nimport scala.reflect.ClassTag\nimport shapeless.test.illTyped\nimport org.scalatest.matchers.should.Matchers\n\nclass CreateTests extends TypedDatasetSuite with Matchers {\n\n import TypedEncoder.usingInjection\n\n test(\"creation using X4 derived DataFrames\") {\n def prop[\n A: TypedEncoder,\n B: TypedEncoder,\n C: TypedEncoder,\n D: TypedEncoder](data: Vector[X4[A, B, C, D]]): Prop = {\n val ds = TypedDataset.create(data)\n TypedDataset.createUnsafe[X4[A, B, C, D]](ds.toDF()).collect().run() ?= data\n }\n\n check(forAll(prop[Int, Char, X2[Option[Country], Country], Int] _))\n check(forAll(prop[X2[Int, Int], Int, Boolean, Vector[Food]] _))\n check(forAll(prop[String, Food, X3[Food, Country, Boolean], Int] _))\n check(forAll(prop[String, Food, X3U[Food, Country, Boolean], Int] _))\n check(forAll(prop[\n Option[Vector[Food]],\n Vector[Vector[X2[Vector[(Person, X1[Char])], Country]]],\n X3[Food, Country, String],\n Vector[(Food, Country)]] _))\n }\n\n test(\"array fields\") {\n def prop[T: Arbitrary: TypedEncoder: ClassTag] = forAll {\n (d1: Array[T], d2: Array[Option[T]], d3: Array[X1[T]], d4: Array[X1[Option[T]]],\n d5: X1[Array[T]]) =>\n TypedDataset.create(Seq(d1)).collect().run().head.sameElements(d1) &&\n TypedDataset.create(Seq(d2)).collect().run().head.sameElements(d2) &&\n TypedDataset.create(Seq(d3)).collect().run().head.sameElements(d3) &&\n TypedDataset.create(Seq(d4)).collect().run().head.sameElements(d4) &&\n TypedDataset.create(Seq(d5)).collect().run().head.a.sameElements(d5.a)\n }\n\n check(prop[Boolean])\n check(prop[Byte])\n check(prop[Short])\n check(prop[Int])\n check(prop[Long])\n check(prop[Float])\n check(prop[Double])\n check(prop[String])\n }\n\n test(\"vector fields\") {\n def prop[T: Arbitrary: TypedEncoder] = forAll {\n (d1: Vector[T], d2: Vector[Option[T]], d3: Vector[X1[T]], d4: Vector[X1[Option[T]]],\n d5: X1[Vector[T]]) =>\n (TypedDataset.create(Seq(d1)).collect().run().head ?= d1) &&\n (TypedDataset.create(Seq(d2)).collect().run().head ?= d2) &&\n (TypedDataset.create(Seq(d3)).collect().run().head ?= d3) &&\n (TypedDataset.create(Seq(d4)).collect().run().head ?= d4) &&\n (TypedDataset.create(Seq(d5)).collect().run().head ?= d5)\n }\n\n check(prop[Boolean])\n check(prop[Byte])\n check(prop[Char])\n check(prop[Short])\n check(prop[Int])\n check(prop[Long])\n check(prop[Float])\n check(prop[Double])\n check(prop[String])\n }\n\n test(\"list fields\") {\n def prop[T: Arbitrary: TypedEncoder] = forAll {\n (d1: List[T], d2: List[Option[T]], d3: List[X1[T]], d4: List[X1[Option[T]]],\n d5: X1[List[T]]) =>\n (TypedDataset.create(Seq(d1)).collect().run().head ?= d1) &&\n (TypedDataset.create(Seq(d2)).collect().run().head ?= d2) &&\n (TypedDataset.create(Seq(d3)).collect().run().head ?= d3) &&\n (TypedDataset.create(Seq(d4)).collect().run().head ?= d4) &&\n (TypedDataset.create(Seq(d5)).collect().run().head ?= d5)\n }\n\n check(prop[Boolean])\n check(prop[Byte])\n check(prop[Char])\n check(prop[Short])\n check(prop[Int])\n check(prop[Long])\n check(prop[Float])\n check(prop[Double])\n check(prop[String])\n }\n\n test(\"Map fields (scala.Predef.Map / scala.collection.immutable.Map)\") {\n def prop[A: Arbitrary: NotCatalystNullable: TypedEncoder, B: Arbitrary: NotCatalystNullable: TypedEncoder] = forAll {\n (d1: Map[A, B], d2: Map[B, A], d3: Map[A, Option[B]],\n d4: Map[A, X1[B]], d5: Map[X1[A], B], d6: Map[X1[A], X1[B]]) =>\n\n (TypedDataset.create(Seq(d1)).collect().run().head ?= d1) &&\n (TypedDataset.create(Seq(d2)).collect().run().head ?= d2) &&\n (TypedDataset.create(Seq(d3)).collect().run().head ?= d3) &&\n (TypedDataset.create(Seq(d4)).collect().run().head ?= d4) &&\n (TypedDataset.create(Seq(d5)).collect().run().head ?= d5) &&\n (TypedDataset.create(Seq(d6)).collect().run().head ?= d6)\n }\n\n check(prop[String, String])\n check(prop[String, Boolean])\n check(prop[String, Byte])\n check(prop[String, Char])\n check(prop[String, Short])\n check(prop[String, Int])\n check(prop[String, Long])\n check(prop[String, Float])\n check(prop[String, Double])\n }\n\n test(\"maps with Option keys should not resolve the TypedEncoder\") {\n val data: Seq[Map[Option[Int], Int]] = Seq(Map(Some(5) -> 5))\n illTyped(\"TypedDataset.create(data)\", \".*could not find implicit value for parameter encoder.*\")\n }\n\n test(\"not aligned columns should throw an exception\") {\n val v = Vector(X2(1,2))\n val df = TypedDataset.create(v).dataset.toDF()\n\n a [IllegalStateException] should be thrownBy {\n TypedDataset.createUnsafe[X1[Int]](df).show().run()\n }\n }\n\n test(\"dataset with different column order\") {\n // e.g. when loading data from partitioned dataset\n // the partition columns get appended to the end of the underlying relation\n def prop[A: Arbitrary: TypedEncoder, B: Arbitrary: TypedEncoder] = forAll {\n (a1: A, b1: B) => {\n val ds = TypedDataset.create(\n Vector((b1, a1))\n ).dataset.toDF(\"b\", \"a\").as[X2[A, B]](TypedExpressionEncoder[X2[A, B]])\n TypedDataset.create(ds).collect().run().head ?= X2(a1, b1)\n\n }\n }\n check(prop[X1[Double], X1[X1[SQLDate]]])\n check(prop[String, Int])\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/DropTest.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\n\nclass DropTest extends TypedDatasetSuite {\n import DropTest._\n\n test(\"fail to compile on missing value\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1, 1, false) :: X(1, 1, false) :: X(1, 10, false) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XMissing] = f.drop[XMissing]('j)\"\"\"\n }\n }\n\n test(\"fail to compile on different column name\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1, 1, false) :: X(1, 1, false) :: X(1, 10, false) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XDifferentColumnName] = f.drop[XDifferentColumnName]('j)\"\"\"\n }\n }\n\n test(\"fail to compile on added column name\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1, 1, false) :: X(1, 1, false) :: X(1, 10, false) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XAdded] = f.drop[XAdded]('j)\"\"\"\n }\n }\n\n test(\"remove column in the middle\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1, 1, false) :: X(1, 1, false) :: X(1, 10, false) :: Nil)\n val fNew: TypedDataset[XGood] = f.drop[XGood]\n\n fNew.collect().run().foreach(xg => assert(xg === XGood(1, false)))\n }\n\n test(\"drop four columns\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d5 = TypedDataset.create(X5(value, value, value, value, value) :: Nil)\n val d4 = d5.drop[X4[A, A, A, A]]\n val d3 = d4.drop[X3[A, A, A]]\n val d2 = d3.drop[X2[A, A]]\n val d1 = d2.drop[X1[A]]\n\n X1(value) ?= d1.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n}\n\nobject DropTest {\n case class X(i: Int, j: Int, k: Boolean)\n case class XMissing(i: Int)\n case class XDifferentColumnName(ij: Int, k: Boolean)\n case class XAdded(i: Int, j: Int, k: Boolean, l: Int)\n case class XGood(i: Int, k: Boolean)\n}\n" }, { "path": "dataset/src/test/scala/frameless/DropTupledTest.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass DropTupledTest extends TypedDatasetSuite {\n test(\"drop five columns\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d5 = TypedDataset.create(X5(value, value, value, value, value) :: Nil)\n val d4 = d5.dropTupled('a) //drops first column\n val d3 = d4.dropTupled('_4) //drops last column\n val d2 = d3.dropTupled('_2) //drops middle column\n val d1 = d2.dropTupled('_2)\n\n Tuple1(value) ?= d1.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n\n test(\"drop first column\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d3 = TypedDataset.create(X3(value, value, value) :: Nil)\n val d2 = d3.dropTupled('a)\n\n (value, value) ?= d2.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n\n test(\"drop middle column\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d3 = TypedDataset.create(X3(value, value, value) :: Nil)\n val d2 = d3.dropTupled('b)\n\n (value, value) ?= d2.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n\n test(\"drop last column\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d3 = TypedDataset.create(X3(value, value, value) :: Nil)\n val d2 = d3.dropTupled('c)\n\n (value, value) ?= d2.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/EncoderTests.scala", "content": "package frameless\n\nimport scala.collection.immutable.Set\n\nimport org.scalatest.matchers.should.Matchers\n\nobject EncoderTests {\n case class Foo(s: Seq[(Int, Int)])\n case class Bar(s: Set[(Int, Int)])\n case class InstantRow(i: java.time.Instant)\n case class DurationRow(d: java.time.Duration)\n case class PeriodRow(p: java.time.Period)\n}\n\nclass EncoderTests extends TypedDatasetSuite with Matchers {\n import EncoderTests._\n\n test(\"It should encode deeply nested collections\") {\n implicitly[TypedEncoder[Seq[Foo]]]\n implicitly[TypedEncoder[Seq[Bar]]]\n implicitly[TypedEncoder[Set[Foo]]]\n }\n\n test(\"It should encode java.time.Instant\") {\n implicitly[TypedEncoder[InstantRow]]\n }\n\n test(\"It should encode java.time.Duration\") {\n implicitly[TypedEncoder[DurationRow]]\n }\n\n test(\"It should encode java.time.Period\") {\n implicitly[TypedEncoder[PeriodRow]]\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ExplodeTests.scala", "content": "package frameless\n\nimport frameless.functions.CatalystExplodableCollection\nimport org.scalacheck.{Arbitrary, Prop}\nimport org.scalacheck.Prop.forAll\nimport org.scalacheck.Prop._\n\nimport scala.reflect.ClassTag\n\nclass ExplodeTests extends TypedDatasetSuite {\n test(\"simple explode test\") {\n val ds = TypedDataset.create(Seq((1,Array(1,2))))\n ds.explode('_2): TypedDataset[(Int,Int)]\n }\n\n test(\"explode on vectors/list/seq\") {\n def prop[F[X] <: Traversable[X] : CatalystExplodableCollection, A: TypedEncoder](xs: List[X1[F[A]]])(implicit arb: Arbitrary[F[A]], enc: TypedEncoder[F[A]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.explode('a).collect().run().toVector\n val scalaResults = xs.flatMap(_.a).map(Tuple1(_)).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Vector, Long] _))\n check(forAll(prop[Seq, Int] _))\n check(forAll(prop[Vector, Char] _))\n check(forAll(prop[Vector, String] _))\n check(forAll(prop[List, Long] _))\n check(forAll(prop[List, Int] _))\n check(forAll(prop[List, Char] _))\n check(forAll(prop[List, String] _))\n }\n\n test(\"explode on arrays\") {\n def prop[A: TypedEncoder: ClassTag](xs: List[X1[Array[A]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.explode('a).collect().run().toVector\n val scalaResults = xs.flatMap(_.a).map(Tuple1(_)).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"explode on maps\") {\n def prop[A: TypedEncoder: ClassTag, B: TypedEncoder: ClassTag](xs: List[X1[Map[A, B]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.explodeMap('a).collect().run().toVector\n val scalaResults = xs.flatMap(_.a.toList).map(t => Tuple1(Tuple2(t._1, t._2))).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Long, String] _))\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[String, Int] _))\n }\n\n test(\"explode on maps preserving other columns\") {\n def prop[K: TypedEncoder: ClassTag, A: TypedEncoder: ClassTag, B: TypedEncoder: ClassTag](xs: List[X2[K, Map[A, B]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.explodeMap('b).collect().run().toVector\n val scalaResults = xs.flatMap { x2 => x2.b.toList.map((x2.a, _)) }.toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Int, Long, String] _))\n check(forAll(prop[String, Int, Long] _))\n check(forAll(prop[Long, String, Int] _))\n }\n\n test(\"explode on maps making sure no key / value naming collision happens\") {\n def prop[K: TypedEncoder: ClassTag, V: TypedEncoder: ClassTag, A: TypedEncoder: ClassTag, B: TypedEncoder: ClassTag](xs: List[X3KV[K, V, Map[A, B]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.explodeMap('c).collect().run().toVector\n val scalaResults = xs.flatMap { x3 => x3.c.toList.map((x3.key, x3.value, _)) }.toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[String, Int, Long, String] _))\n check(forAll(prop[Long, String, Int, Long] _))\n check(forAll(prop[Int, Long, String, Int] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/FilterTests.scala", "content": "package frameless\n\nimport org.scalatest.matchers.should.Matchers\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nfinal class FilterTests extends TypedDatasetSuite with Matchers {\n test(\"filter('a == lit(b))\") {\n def prop[A: TypedEncoder](elem: A, data: Vector[X1[A]])(implicit ex1: TypedEncoder[X1[A]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col('a)\n\n val dataset2 = dataset.filter(A === elem).collect().run().toVector\n val data2 = data.filter(_.a == elem)\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"filter('a =!= lit(b))\") {\n def prop[A: TypedEncoder](elem: A, data: Vector[X1[A]])(implicit ex1: TypedEncoder[X1[A]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col('a)\n\n val dataset2 = dataset.filter(A =!= elem).collect().run().toVector\n val data2 = data.filter(_.a != elem)\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Char] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[Vector[SQLTimestamp]] _))\n }\n\n test(\"filter('a =!= 'b)\") {\n def prop[A: TypedEncoder](data: Vector[X2[A, A]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col('a)\n val B = dataset.col('b)\n\n val dataset2 = dataset.filter(A =!= B).collect().run().toVector\n val data2 = data.filter(x => x.a != x.b)\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Char] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[Vector[SQLTimestamp]] _))\n }\n\n test(\"filter('a =!= 'b\") {\n def prop[A: TypedEncoder](elem: A, data: Vector[X2[A,A]]): Prop = {\n val dataset = TypedDataset.create(data)\n val cA = dataset.col('a)\n val cB = dataset.col('b)\n\n val dataset2 = dataset.filter(cA =!= cB).collect().run().toVector\n val data2 = data.filter(x => x.a != x.b )\n\n (dataset2 ?= data2).&&(dataset.filter(cA =!= cA).count().run() ?= 0)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Char] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[Vector[SQLTimestamp]] _))\n }\n\n test(\"filter with arithmetic expressions: addition\") {\n check(forAll { (data: Vector[X1[Int]]) =>\n val ds = TypedDataset.create(data)\n val res = ds.filter((ds('a) + 1) === (ds('a) + 1)).collect().run().toVector\n res ?= data\n })\n }\n\n test(\"filter with values (not columns): addition\") {\n check(forAll { (data: Vector[X1[Int]], const: Int) =>\n val ds = TypedDataset.create(data)\n val res = ds.filter(ds('a) > const).collect().run().toVector\n res ?= data.filter(_.a > const)\n })\n }\n\n test(\"filter with arithmetic expressions: multiplication\") {\n val t = X1(1) :: X1(2) :: X1(3) :: Nil\n val tds: TypedDataset[X1[Int]] = TypedDataset.create(t)\n\n assert(tds.filter(tds('a) * 2 === 2).collect().run().toVector === Vector(X1(1)))\n assert(tds.filter(tds('a) * 3 === 3).collect().run().toVector === Vector(X1(1)))\n }\n\n test(\"Option equality/inequality for columns\") {\n def prop[A <: Option[_] : TypedEncoder](a: A, b: A): Prop = {\n val data = X2(a, b) :: X2(a, a) :: Nil\n val dataset = TypedDataset.create(data)\n val A = dataset.col('a)\n val B = dataset.col('b)\n\n (data.filter(x => x.a == x.b).toSet ?= dataset.filter(A === B).collect().run().toSet).\n &&(data.filter(x => x.a != x.b).toSet ?= dataset.filter(A =!= B).collect().run().toSet).\n &&(data.filter(x => x.a == None).toSet ?= dataset.filter(A.isNone).collect().run().toSet).\n &&(data.filter(x => x.a == None).toSet ?= dataset.filter(A.isNotNone === false).collect().run().toSet)\n }\n\n check(forAll(prop[Option[Int]] _))\n check(forAll(prop[Option[Boolean]] _))\n check(forAll(prop[Option[SQLDate]] _))\n check(forAll(prop[Option[SQLTimestamp]] _))\n check(forAll(prop[Option[X1[String]]] _))\n check(forAll(prop[Option[X1[X1[String]]]] _))\n check(forAll(prop[Option[X1[X1[Vector[Option[Int]]]]]] _))\n }\n\n test(\"Option equality/inequality for lit\") {\n def prop[A <: Option[_] : TypedEncoder](a: A, b: A, cLit: A): Prop = {\n val data = X2(a, b) :: X2(a, cLit) :: Nil\n val dataset = TypedDataset.create(data)\n val colA = dataset.col('a)\n\n (data.filter(x => x.a == cLit).toSet ?= dataset.filter(colA === cLit).collect().run().toSet).\n &&(data.filter(x => x.a != cLit).toSet ?= dataset.filter(colA =!= cLit).collect().run().toSet).\n &&(data.filter(x => x.a == None).toSet ?= dataset.filter(colA.isNone).collect().run().toSet).\n &&(data.filter(x => x.a == None).toSet ?= dataset.filter(colA.isNotNone === false).collect().run().toSet)\n }\n\n check(forAll(prop[Option[Int]] _))\n check(forAll(prop[Option[Boolean]] _))\n check(forAll(prop[Option[SQLDate]] _))\n check(forAll(prop[Option[SQLTimestamp]] _))\n check(forAll(prop[Option[String]] _))\n check(forAll(prop[Option[X1[String]]] _))\n check(forAll(prop[Option[X1[X1[String]]]] _))\n check(forAll(prop[Option[X1[X1[Vector[Option[Int]]]]]] _))\n }\n\n test(\"Option content filter\") {\n val data = (Option(1L), Option(2L)) :: (Option(0L), Option(1L)) :: (None, None) :: Nil\n\n val ds = TypedDataset.create(data)\n\n val l = functions.lit[Long, (Option[Long], Option[Long])](0L)\n val exists = ds('_1).isSome[Long](_ <= l)\n val forall = ds('_1).isSomeOrNone[Long](_ <= l)\n\n ds.select(exists).collect().run() shouldEqual Seq(false, true, false)\n ds.select(forall).collect().run() shouldEqual Seq(false, true, true)\n\n ds.filter(exists).collect().run() shouldEqual Seq(Option(0L) -> Option(1L))\n\n ds.filter(forall).collect().run() shouldEqual Seq(\n Option(0L) -> Option(1L), (None -> None))\n }\n\n test(\"filter with isin values\") {\n def prop[A: TypedEncoder](data: Vector[X1[A]], values: Vector[A])(implicit a : CatalystIsin[A]): Prop = {\n val ds = TypedDataset.create(data)\n val res = ds.filter(ds('a).isin(values:_*)).collect().run().toVector\n res ?= data.filter(d => values.contains(d.a))\n }\n\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/FlattenTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.forAll\nimport org.scalacheck.Prop._\n\n\nclass FlattenTests extends TypedDatasetSuite {\n test(\"simple flatten test\") {\n val ds: TypedDataset[(Int,Option[Int])] = TypedDataset.create(Seq((1,Option(1))))\n ds.flattenOption('_2): TypedDataset[(Int,Int)]\n }\n\n test(\"different Optional types\") {\n def prop[A: TypedEncoder](xs: List[X1[Option[A]]]): Prop = {\n val tds: TypedDataset[X1[Option[A]]] = TypedDataset.create(xs)\n\n val framelessResults: Seq[Tuple1[A]] = tds.flattenOption('a).collect().run().toVector\n val scalaResults = xs.flatMap(_.a).map(Tuple1(_)).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Char] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/GroupByTests.scala", "content": "package frameless\n\nimport frameless.functions.aggregate._\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass GroupByTests extends TypedDatasetSuite {\n test(\"groupByMany('a).agg(sum('b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder,\n Out: TypedEncoder : Numeric\n ](data: List[X2[A, B]])(\n implicit\n summable: CatalystSummable[B, Out],\n widen: B => Out\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val datasetSumByA = dataset.groupByMany(A).agg(sum(B)).collect().run.toVector.sortBy(_._1)\n val sumByA = data.groupBy(_.a).map { case (k, v) => k -> v.map(_.b).map(widen).sum }.toVector.sortBy(_._1)\n\n datasetSumByA ?= sumByA\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"agg(sum('a))\") {\n def prop[A: TypedEncoder : Numeric](data: List[X1[A]])(\n implicit\n summable: CatalystSummable[A, A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetSum = dataset.agg(sum(A)).collect().run().toVector\n val listSum = data.map(_.a).sum\n\n datasetSum ?= Vector(listSum)\n }\n\n check(forAll(prop[Long] _))\n }\n\n test(\"agg(sum('a), sum('b))\") {\n def prop[\n A: TypedEncoder : Numeric,\n B: TypedEncoder : Numeric\n ](data: List[X2[A, B]])(\n implicit\n as: CatalystSummable[A, A],\n bs: CatalystSummable[B, B]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val datasetSum = dataset.agg(sum(A), sum(B)).collect().run().toVector\n val listSumA = data.map(_.a).sum\n val listSumB = data.map(_.b).sum\n\n datasetSum ?= Vector((listSumA, listSumB))\n }\n\n check(forAll(prop[Long, Long] _))\n }\n\n test(\"agg(sum('a), sum('b), sum('c))\") {\n def prop[\n A: TypedEncoder : Numeric,\n B: TypedEncoder : Numeric,\n C: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]])(\n implicit\n as: CatalystSummable[A, A],\n bs: CatalystSummable[B, B],\n cs: CatalystSummable[C, C]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val datasetSum = dataset.agg(sum(A), sum(B), sum(C)).collect().run().toVector\n val listSumA = data.map(_.a).sum\n val listSumB = data.map(_.b).sum\n val listSumC = data.map(_.c).sum\n\n datasetSum ?= Vector((listSumA, listSumB, listSumC))\n }\n\n check(forAll(prop[Long, Long, Long] _))\n }\n\n test(\"agg(sum('a), sum('b), min('c), max('d))\") {\n def prop[\n A: TypedEncoder : Numeric,\n B: TypedEncoder : Numeric,\n C: TypedEncoder : Numeric,\n D: TypedEncoder : Numeric\n ](data: List[X4[A, B, C, D]])(\n implicit\n as: CatalystSummable[A, A],\n bs: CatalystSummable[B, B],\n co: CatalystOrdered[C],\n fo: CatalystOrdered[D]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n\n val datasetSum = dataset.agg(sum(A), sum(B), min(C), max(D)).collect().run().toVector\n val listSumA = data.map(_.a).sum\n val listSumB = data.map(_.b).sum\n val listMinC = if(data.isEmpty) implicitly[Numeric[C]].fromInt(0) else data.map(_.c).min\n val listMaxD = if(data.isEmpty) implicitly[Numeric[D]].fromInt(0) else data.map(_.d).max\n\n datasetSum ?= Vector(if (data.isEmpty) null else (listSumA, listSumB, listMinC, listMaxD))\n }\n\n check(forAll(prop[Long, Long, Long, Int] _))\n check(forAll(prop[Long, Long, Short, Short] _))\n check(forAll(prop[Long, Long, Double, BigDecimal] _))\n }\n\n test(\"groupBy('a).agg(sum('b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder,\n Out: TypedEncoder : Numeric\n ](data: List[X2[A, B]])(\n implicit\n summable: CatalystSummable[B, Out],\n widen: B => Out\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val datasetSumByA = dataset.groupBy(A).agg(sum(B)).collect().run.toVector.sortBy(_._1)\n val sumByA = data.groupBy(_.a).mapValues(_.map(_.b).map(widen).sum).toVector.sortBy(_._1)\n\n datasetSumByA ?= sumByA\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"groupBy('a).mapGroups('a, sum('b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Numeric\n ](data: List[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val datasetSumByA = dataset.groupBy(A)\n .deserialized.mapGroups { case (a, xs) => (a, xs.map(_.b).sum) }\n .collect().run().toVector.sortBy(_._1)\n val sumByA = data.groupBy(_.a).mapValues(_.map(_.b).sum).toVector.sortBy(_._1)\n\n datasetSumByA ?= sumByA\n }\n\n check(forAll(prop[Int, Long] _))\n }\n\n test(\"groupBy('a).agg(sum('b), sum('c)) to groupBy('a).agg(sum('a), sum('b), sum('a), sum('b), sum('a))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder,\n C: TypedEncoder,\n OutB: TypedEncoder : Numeric,\n OutC: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]])(\n implicit\n summableB: CatalystSummable[B, OutB],\n summableC: CatalystSummable[C, OutC],\n widenb: B => OutB,\n widenc: C => OutC\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumBC = dataset\n .groupBy(A)\n .agg(sum(B), sum(C))\n .collect().run.toVector.sortBy(_._1)\n\n val scalaSumBC = data.groupBy(_.a).mapValues { xs =>\n (xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum)\n }.toVector.map {\n case (a, (b, c)) => (a, b, c)\n }.sortBy(_._1)\n\n val framelessSumBCB = dataset\n .groupBy(A)\n .agg(sum(B), sum(C), sum(B))\n .collect().run.toVector.sortBy(_._1)\n\n val scalaSumBCB = data.groupBy(_.a).mapValues { xs =>\n (xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum, xs.map(_.b).map(widenb).sum)\n }.toVector.map {\n case (a, (b1, c, b2)) => (a, b1, c, b2)\n }.sortBy(_._1)\n\n val framelessSumBCBC = dataset\n .groupBy(A)\n .agg(sum(B), sum(C), sum(B), sum(C))\n .collect().run.toVector.sortBy(_._1)\n\n val scalaSumBCBC = data.groupBy(_.a).mapValues { xs =>\n (xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum, xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum)\n }.toVector.map {\n case (a, (b1, c1, b2, c2)) => (a, b1, c1, b2, c2)\n }.sortBy(_._1)\n\n val framelessSumBCBCB = dataset\n .groupBy(A)\n .agg(sum(B), sum(C), sum(B), sum(C), sum(B))\n .collect().run.toVector.sortBy(_._1)\n\n val scalaSumBCBCB = data.groupBy(_.a).mapValues { xs =>\n (xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum, xs.map(_.b).map(widenb).sum, xs.map(_.c).map(widenc).sum, xs.map(_.b).map(widenb).sum)\n }.toVector.map {\n case (a, (b1, c1, b2, c2, b3)) => (a, b1, c1, b2, c2, b3)\n }.sortBy(_._1)\n\n (framelessSumBC ?= scalaSumBC)\n .&&(framelessSumBCB ?= scalaSumBCB)\n .&&(framelessSumBCBC ?= scalaSumBCBC)\n .&&(framelessSumBCBCB ?= scalaSumBCBCB)\n }\n\n check(forAll(prop[String, Long, BigDecimal, Long, BigDecimal] _))\n }\n\n test(\"groupBy('a, 'b).agg(sum('c)) to groupBy('a, 'b).agg(sum('c),sum('c),sum('c),sum('c),sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n OutC: TypedEncoder: Numeric\n ](data: List[X3[A, B, C]])(\n implicit\n summableC: CatalystSummable[C, OutC],\n widenc: C => OutC\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumC = dataset\n .groupBy(A,B)\n .agg(sum(C))\n .collect().run.toVector.sortBy(x => (x._1,x._2))\n\n val scalaSumC = data.groupBy(x => (x.a,x.b)).mapValues { xs =>\n xs.map(_.c).map(widenc).sum\n }.toVector.map { case ((a, b), c) => (a, b, c) }.sortBy(x => (x._1,x._2))\n\n val framelessSumCC = dataset\n .groupBy(A,B)\n .agg(sum(C), sum(C))\n .collect().run.toVector.sortBy(x => (x._1,x._2))\n\n val scalaSumCC = data.groupBy(x => (x.a,x.b)).mapValues { xs =>\n val s = xs.map(_.c).map(widenc).sum; (s,s)\n }.toVector.map { case ((a, b), (c1, c2)) => (a, b, c1, c2) }.sortBy(x => (x._1,x._2))\n\n val framelessSumCCC = dataset\n .groupBy(A,B)\n .agg(sum(C), sum(C), sum(C))\n .collect().run.toVector.sortBy(x => (x._1,x._2))\n\n val scalaSumCCC = data.groupBy(x => (x.a,x.b)).mapValues { xs =>\n val s = xs.map(_.c).map(widenc).sum; (s,s,s)\n }.toVector.map { case ((a, b), (c1, c2, c3)) => (a, b, c1, c2, c3) }.sortBy(x => (x._1,x._2))\n\n val framelessSumCCCC = dataset\n .groupBy(A,B)\n .agg(sum(C), sum(C), sum(C), sum(C))\n .collect().run.toVector.sortBy(x => (x._1,x._2))\n\n val scalaSumCCCC = data.groupBy(x => (x.a,x.b)).mapValues { xs =>\n val s = xs.map(_.c).map(widenc).sum; (s,s,s,s)\n }.toVector.map { case ((a, b), (c1, c2, c3, c4)) => (a, b, c1, c2, c3, c4) }.sortBy(x => (x._1,x._2))\n\n val framelessSumCCCCC = dataset\n .groupBy(A,B)\n .agg(sum(C), sum(C), sum(C), sum(C), sum(C))\n .collect().run.toVector.sortBy(x => (x._1,x._2))\n\n val scalaSumCCCCC = data.groupBy(x => (x.a,x.b)).mapValues { xs =>\n val s = xs.map(_.c).map(widenc).sum; (s,s,s,s,s)\n }.toVector.map { case ((a, b), (c1, c2, c3, c4, c5)) => (a, b, c1, c2, c3, c4, c5) }.sortBy(x => (x._1,x._2))\n\n (framelessSumC ?= scalaSumC) &&\n (framelessSumCC ?= scalaSumCC) &&\n (framelessSumCCC ?= scalaSumCCC) &&\n (framelessSumCCCC ?= scalaSumCCCC) &&\n (framelessSumCCCCC ?= scalaSumCCCCC)\n }\n\n check(forAll(prop[String, Long, BigDecimal, BigDecimal] _))\n }\n\n test(\"groupBy('a, 'b).agg(sum('c), sum('d))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n D: TypedEncoder,\n OutC: TypedEncoder : Numeric,\n OutD: TypedEncoder : Numeric\n ](data: List[X4[A, B, C, D]])(\n implicit\n summableC: CatalystSummable[C, OutC],\n summableD: CatalystSummable[D, OutD],\n widenc: C => OutC,\n widend: D => OutD\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n\n val datasetSumByAB = dataset\n .groupBy(A, B)\n .agg(sum(C), sum(D))\n .collect().run.toVector.sortBy(x => (x._1, x._2))\n\n val sumByAB = data.groupBy(x => (x.a, x.b)).mapValues { xs =>\n (xs.map(_.c).map(widenc).sum, xs.map(_.d).map(widend).sum)\n }.toVector.map {\n case ((a, b), (c, d)) => (a, b, c, d)\n }.sortBy(x => (x._1, x._2))\n\n datasetSumByAB ?= sumByAB\n }\n\n check(forAll(prop[Byte, Int, Long, BigDecimal, Long, BigDecimal] _))\n }\n\n test(\"groupBy('a, 'b).mapGroups('a, 'b, sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val datasetSumByAB = dataset\n .groupBy(A, B)\n .deserialized.mapGroups { case ((a, b), xs) => (a, b, xs.map(_.c).sum) }\n .collect().run().toVector.sortBy(x => (x._1, x._2))\n\n val sumByAB = data.groupBy(x => (x.a, x.b))\n .mapValues { xs => xs.map(_.c).sum }\n .toVector.map { case ((a, b), c) => (a, b, c) }.sortBy(x => (x._1, x._2))\n\n datasetSumByAB ?= sumByAB\n }\n\n check(forAll(prop[Byte, Int, Long] _))\n }\n\n test(\"groupBy('a).mapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder: Ordering,\n B: TypedEncoder: Ordering\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .groupBy(A)\n .deserialized.mapGroups((a, xs) => (a, xs.toVector.sorted))\n .collect().run.toMap\n\n val dataGrouped = data.groupBy(_.a).map { case (k, v) => k -> v.sorted }\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"groupBy('a).flatMapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .groupBy(A)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run\n .sorted\n\n val dataGrouped = data\n .groupBy(_.a).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"groupBy('a, 'b).flatMapGroups((('a,'b) toVector((('a,'b), 'c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Ordering\n ](data: Vector[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val cA = dataset.col[A]('a)\n val cB = dataset.col[B]('b)\n\n val datasetGrouped = dataset\n .groupBy(cA, cB)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run()\n .sorted\n\n val dataGrouped = data\n .groupBy(t => (t.a,t.b)).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short], Long] _))\n check(forAll(prop[Option[Short], Short, Int] _))\n check(forAll(prop[X1[Option[Short]], Short, Byte] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/InjectionTests.scala", "content": "package frameless\n\nimport frameless.CollectTests.prop\nimport org.scalacheck._\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\n\nsealed trait Country\ncase object France extends Country\ncase object Russia extends Country\n\nobject Country {\n implicit val arbitrary: Arbitrary[Country] =\n Arbitrary(Arbitrary.arbitrary[Boolean].map(injection.invert))\n\n implicit val injection: Injection[Country, Boolean] =\n Injection(France.==, if (_) France else Russia)\n}\n\nsealed trait Food\ncase object Burger extends Food\ncase object Pasta extends Food\ncase object Rice extends Food\n\nobject Food {\n implicit val arbitrary: Arbitrary[Food] =\n Arbitrary(Arbitrary.arbitrary[Int].map(i => injection.invert(Math.abs(i % 3))))\n\n implicit val injection: Injection[Food, Int] =\n Injection(\n {\n case Burger => 0\n case Pasta => 1\n case Rice => 2\n },\n {\n case 0 => Burger\n case 1 => Pasta\n case 2 => Rice\n }\n )\n}\n\n// Supposingly coming from a java lib, shapeless can't derive stuff for this one :(\nclass LocalDateTime {\n var instant: Long = _\n\n override def equals(o: Any): Boolean =\n o.isInstanceOf[LocalDateTime] && o.asInstanceOf[LocalDateTime].instant == instant\n}\n\nobject LocalDateTime {\n implicit val arbitrary: Arbitrary[LocalDateTime] =\n Arbitrary(Arbitrary.arbitrary[Long].map(injection.invert))\n\n implicit val injection: Injection[LocalDateTime, Long] =\n Injection(\n _.instant,\n long => { val ldt = new LocalDateTime; ldt.instant = long; ldt }\n )\n}\n\ncase class Person(age: Int, name: String)\n\nobject Person {\n val tupled = (Person.apply _).tupled\n\n implicit val arbitrary: Arbitrary[Person] =\n Arbitrary(Arbitrary.arbTuple2[Int, String].arbitrary.map(tupled))\n\n implicit val injection: Injection[Person, (Int, String)] =\n Injection(p => unapply(p).get, tupled)\n}\n\ncase class I[A](value: A)\n\nobject I {\n implicit def injection[A]: Injection[I[A], A] = Injection(_.value, I(_))\n implicit def typedEncoder[A: TypedEncoder]: TypedEncoder[I[A]] = TypedEncoder.usingInjection[I[A], A]\n implicit def arbitrary[A: Arbitrary]: Arbitrary[I[A]] = Arbitrary(Arbitrary.arbitrary[A].map(I(_)))\n}\n\nsealed trait Employee\ncase object Casual extends Employee\ncase object PartTime extends Employee\ncase object FullTime extends Employee\n\nobject Employee {\n implicit val arbitrary: Arbitrary[Employee] =\n Arbitrary(Gen.oneOf(Casual, PartTime, FullTime))\n}\n\nsealed trait Maybe\ncase object Nothing extends Maybe\ncase class Just(get: Int) extends Maybe\n\nsealed trait Switch\nobject Switch {\n case object Off extends Switch\n case object On extends Switch\n\n implicit val arbitrary: Arbitrary[Switch] =\n Arbitrary(Gen.oneOf(Off, On))\n}\n\nsealed trait Pixel\ncase class Red() extends Pixel\ncase class Green() extends Pixel\ncase class Blue() extends Pixel\n\nobject Pixel {\n implicit val arbitrary: Arbitrary[Pixel] =\n Arbitrary(Gen.oneOf(Red(), Green(), Blue()))\n}\n\nsealed trait Connection[+A]\ncase object Closed extends Connection[Nothing]\ncase object Open extends Connection[Nothing]\n\nobject Connection {\n implicit def arbitrary[A]: Arbitrary[Connection[A]] =\n Arbitrary(Gen.oneOf(Closed, Open))\n}\n\nsealed abstract class Vehicle(colour: String)\ncase object Car extends Vehicle(\"red\")\ncase object Bike extends Vehicle(\"blue\")\n\nobject Vehicle {\n implicit val arbitrary: Arbitrary[Vehicle] =\n Arbitrary(Gen.oneOf(Car, Bike))\n}\n\nclass InjectionTests extends TypedDatasetSuite {\n test(\"Injection based encoders\") {\n check(forAll(prop[Country] _))\n check(forAll(prop[LocalDateTime] _))\n check(forAll(prop[Food] _))\n check(forAll(prop[X1[Country]] _))\n check(forAll(prop[X1[LocalDateTime]] _))\n check(forAll(prop[X1[Food]] _))\n check(forAll(prop[X1[X1[Country]]] _))\n check(forAll(prop[X1[X1[LocalDateTime]]] _))\n check(forAll(prop[X1[X1[Food]]] _))\n check(forAll(prop[X2[Country, X2[LocalDateTime, Food]]] _))\n check(forAll(prop[X3[Country, LocalDateTime, Food]] _))\n check(forAll(prop[X3U[Country, LocalDateTime, Food]] _))\n\n check(forAll(prop[I[Int]] _))\n check(forAll(prop[I[Option[Int]]] _))\n check(forAll(prop[I[I[Int]]] _))\n check(forAll(prop[I[I[Option[Int]]]] _))\n\n check(forAll(prop[I[X1[Int]]] _))\n check(forAll(prop[I[I[X1[Int]]]] _))\n check(forAll(prop[I[I[Option[X1[Int]]]]] _))\n\n check(forAll(prop[Option[I[Int]]] _))\n check(forAll(prop[Option[I[X1[Int]]]] _))\n\n assert(TypedEncoder[I[Int]].catalystRepr == TypedEncoder[Int].catalystRepr)\n assert(TypedEncoder[I[I[Int]]].catalystRepr == TypedEncoder[Int].catalystRepr)\n\n assert(TypedEncoder[I[Option[Int]]].nullable)\n }\n\n test(\"TypedEncoder[Person] is ambiguous\") {\n illTyped(\"implicitly[TypedEncoder[Person]]\", \"ambiguous implicit values.*\")\n }\n\n test(\"Resolve ambiguity by importing usingInjection\") {\n import TypedEncoder.usingInjection\n\n check(forAll(prop[X1[Person]] _))\n check(forAll(prop[X1[X1[Person]]] _))\n check(forAll(prop[X2[Person, Person]] _))\n check(forAll(prop[Person] _))\n\n assert(TypedEncoder[Person].catalystRepr == TypedEncoder[(Int, String)].catalystRepr)\n }\n\n test(\"Resolve ambiguity by importing usingDerivation\") {\n import TypedEncoder.usingDerivation\n assert(implicitly[TypedEncoder[Person]].isInstanceOf[RecordEncoder[Person, _, _]])\n check(forAll(prop[Person] _))\n }\n\n test(\"TypedEncoder[Employee] implicit is missing\") {\n illTyped(\n \"implicitly[TypedEncoder[Employee]]\",\n \"could not find implicit value for parameter e.*\"\n )\n }\n\n test(\"Resolve missing implicit by deriving Injection instance\") {\n import frameless.TypedEncoder.injections._\n\n check(forAll(prop[X1[Employee]] _))\n check(forAll(prop[X1[X1[Employee]]] _))\n check(forAll(prop[X2[Employee, Employee]] _))\n check(forAll(prop[Employee] _))\n\n assert(TypedEncoder[Employee].catalystRepr == TypedEncoder[String].catalystRepr)\n }\n\n test(\"TypedEncoder[Maybe] cannot be derived\") {\n import frameless.TypedEncoder.injections._\n\n illTyped(\n \"implicitly[TypedEncoder[Maybe]]\",\n \"could not find implicit value for parameter e.*\"\n )\n }\n\n test(\"Derive encoder for type with data constructors defined in the companion object\") {\n import frameless.TypedEncoder.injections._\n\n check(forAll(prop[X1[Switch]] _))\n check(forAll(prop[X1[X1[Switch]]] _))\n check(forAll(prop[X2[Switch, Switch]] _))\n check(forAll(prop[Switch] _))\n\n assert(TypedEncoder[Switch].catalystRepr == TypedEncoder[String].catalystRepr)\n }\n\n test(\"Derive encoder for type with data constructors defined as parameterless case classes\") {\n import frameless.TypedEncoder.injections._\n\n check(forAll(prop[X1[Pixel]] _))\n check(forAll(prop[X1[X1[Pixel]]] _))\n check(forAll(prop[X2[Pixel, Pixel]] _))\n check(forAll(prop[Pixel] _))\n\n assert(TypedEncoder[Pixel].catalystRepr == TypedEncoder[String].catalystRepr)\n }\n\n test(\"Derive encoder for phantom type\") {\n import frameless.TypedEncoder.injections._\n\n check(forAll(prop[X1[Connection[Int]]] _))\n check(forAll(prop[X1[X1[Connection[Int]]]] _))\n check(forAll(prop[X2[Connection[Int], Connection[Int]]] _))\n check(forAll(prop[Connection[Int]] _))\n\n assert(TypedEncoder[Connection[Int]].catalystRepr == TypedEncoder[String].catalystRepr)\n }\n\n test(\"Derive encoder for ADT with abstract class as the base type\") {\n import frameless.TypedEncoder.injections._\n\n check(forAll(prop[X1[Vehicle]] _))\n check(forAll(prop[X1[X1[Vehicle]]] _))\n check(forAll(prop[X2[Vehicle, Vehicle]] _))\n check(forAll(prop[Vehicle] _))\n\n assert(TypedEncoder[Vehicle].catalystRepr == TypedEncoder[String].catalystRepr)\n }\n\n test(\"apply method of derived Injection instance produces the correct string\") {\n import frameless.TypedEncoder.injections._\n\n assert(implicitly[Injection[Employee, String]].apply(Casual) === \"Casual\")\n assert(implicitly[Injection[Switch, String]].apply(Switch.On) === \"On\")\n assert(implicitly[Injection[Pixel, String]].apply(Blue()) === \"Blue\")\n assert(implicitly[Injection[Connection[Int], String]].apply(Open) === \"Open\")\n assert(implicitly[Injection[Vehicle, String]].apply(Bike) === \"Bike\")\n }\n\n test(\"invert method of derived Injection instance produces the correct value\") {\n import frameless.TypedEncoder.injections._\n\n assert(implicitly[Injection[Employee, String]].invert(\"Casual\") === Casual)\n assert(implicitly[Injection[Switch, String]].invert(\"On\") === Switch.On)\n assert(implicitly[Injection[Pixel, String]].invert(\"Blue\") === Blue())\n assert(implicitly[Injection[Connection[Int], String]].invert(\"Open\") === Open)\n assert(implicitly[Injection[Vehicle, String]].invert(\"Bike\") === Bike)\n }\n\n test(\n \"invert method of derived Injection instance should throw exception if string does not match data constructor names\"\n ) {\n import frameless.TypedEncoder.injections._\n\n val caught = intercept[IllegalArgumentException] {\n implicitly[Injection[Employee, String]].invert(\"cassual\")\n }\n\n assert(\n caught.getMessage ===\n \"Cannot construct a value of type CNil: cassual did not match data constructor names\"\n )\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/IsValueClassTests.scala", "content": "package frameless\n\nimport shapeless.Refute\nimport shapeless.test.illTyped\n\nimport org.scalatest.funsuite.AnyFunSuite\nimport org.scalatest.matchers.should.Matchers\n\nfinal class IsValueClassTests extends AnyFunSuite with Matchers {\n test(\"Case class is not Value class\") {\n illTyped(\"IsValueClass[P]\")\n illTyped(\"IsValueClass[Q]\")\n }\n\n test(\"Scala value type is not Value class (excluded)\") {\n illTyped(\"implicitly[IsValueClass[Double]]\")\n illTyped(\"implicitly[IsValueClass[Float]]\")\n illTyped(\"implicitly[IsValueClass[Long]]\")\n illTyped(\"implicitly[IsValueClass[Int]]\")\n illTyped(\"implicitly[IsValueClass[Char]]\")\n illTyped(\"implicitly[IsValueClass[Short]]\")\n illTyped(\"implicitly[IsValueClass[Byte]]\")\n illTyped(\"implicitly[IsValueClass[Unit]]\")\n illTyped(\"implicitly[IsValueClass[Boolean]]\")\n illTyped(\"implicitly[IsValueClass[BigDecimal]]\")\n }\n\n test(\"Value class evidence\") {\n implicitly[IsValueClass[RecordEncoderTests.Name]]\n illTyped(\"implicitly[Refute[IsValueClass[RecordEncoderTests.Name]]]\")\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/JobTests.scala", "content": "package frameless\n\nimport org.scalacheck.Arbitrary\nimport org.scalatest.BeforeAndAfterAll\nimport org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks\nimport org.scalatest.freespec.AnyFreeSpec\nimport org.scalatest.matchers.should.Matchers\n\n\nclass JobTests extends AnyFreeSpec with BeforeAndAfterAll with SparkTesting with ScalaCheckDrivenPropertyChecks with Matchers {\n\n \"map\" - {\n \"identity\" in {\n def check[T](implicit arb: Arbitrary[T]) = forAll {\n t: T => Job(t).map(identity).run() shouldEqual Job(t).run()\n }\n\n check[Int]\n }\n\n val f1: Int => Int = _ + 1\n val f2: Int => Int = (i: Int) => i * i\n\n \"composition\" in forAll {\n i: Int => Job(i).map(f1).map(f2).run() shouldEqual Job(i).map(f1 andThen f2).run()\n }\n }\n\n \"flatMap\" - {\n val f1: Int => Job[Int] = (i: Int) => Job(i + 1)\n val f2: Int => Job[Int] = (i: Int) => Job(i * i)\n\n \"left identity\" in forAll {\n i: Int => Job(i).flatMap(f1).run() shouldEqual f1(i).run()\n }\n\n \"right identity\" in forAll {\n i: Int => Job(i).flatMap(i => Job.apply(i)).run() shouldEqual Job(i).run()\n }\n\n \"associativity\" in forAll {\n i: Int => Job(i).flatMap(f1).flatMap(f2).run() shouldEqual Job(i).flatMap(ii => f1(ii).flatMap(f2)).run()\n }\n }\n\n \"properties\" - {\n \"read back\" in forAll {\n (k:String, v: String) =>\n val scopedKey = \"frameless.tests.\" + k\n Job(1).withLocalProperty(scopedKey,v).run()\n sc.getLocalProperty(scopedKey) shouldBe v\n }\n }\n}" }, { "path": "dataset/src/test/scala/frameless/JoinTests.scala", "content": "package frameless\n\nimport org.apache.spark.sql.types.{StructField, StructType}\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass JoinTests extends TypedDatasetSuite {\n test(\"ab.joinCross(ac)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val joinedDs = leftDs\n .joinCross(rightDs)\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val joined = {\n for {\n ab <- left\n ac <- right\n } yield (ab, ac)\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= StructType(Seq(\n StructField(\"_1\", leftDs.schema, nullable = false),\n StructField(\"_2\", rightDs.schema, nullable = false)))\n\n (joined.sorted ?= joinedData) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinFull(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val joinedDs = leftDs\n .joinFull(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val rightKeys = right.map(_.a).toSet\n val leftKeys = left.map(_.a).toSet\n val joined = {\n for {\n ab <- left\n ac <- right if ac.a == ab.a\n } yield (Some(ab), Some(ac))\n }.toVector ++ {\n for {\n ab <- left if !rightKeys.contains(ab.a)\n } yield (Some(ab), None)\n }.toVector ++ {\n for {\n ac <- right if !leftKeys.contains(ac.a)\n } yield (None, Some(ac))\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= StructType(Seq(\n StructField(\"_1\", leftDs.schema, nullable = true),\n StructField(\"_2\", rightDs.schema, nullable = true)))\n\n (joined.sorted ?= joinedData) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinInner(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val joinedDs = leftDs\n .joinInner(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val joined = {\n for {\n ab <- left\n ac <- right if ac.a == ab.a\n } yield (ab, ac)\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= StructType(Seq(\n StructField(\"_1\", leftDs.schema, nullable = false),\n StructField(\"_2\", rightDs.schema, nullable = false)))\n\n (joined.sorted ?= joinedData) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinLeft(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val joinedDs = leftDs\n .joinLeft(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val rightKeys = right.map(_.a).toSet\n val joined = {\n for {\n ab <- left\n ac <- right if ac.a == ab.a\n } yield (ab, Some(ac))\n }.toVector ++ {\n for {\n ab <- left if !rightKeys.contains(ab.a)\n } yield (ab, None)\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= StructType(Seq(\n StructField(\"_1\", leftDs.schema, nullable = false),\n StructField(\"_2\", rightDs.schema, nullable = true)))\n\n (joined.sorted ?= joinedData) && (joinedData.map(_._1).toSet ?= left.toSet) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinLeftAnti(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val rightKeys = right.map(_.a).toSet\n val joinedDs = leftDs\n .joinLeftAnti(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val joined = {\n for {\n ab <- left if !rightKeys.contains(ab.a)\n } yield ab\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= leftDs.schema\n\n (joined.sorted ?= joinedData) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinLeftSemi(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val rightKeys = right.map(_.a).toSet\n val joinedDs = leftDs\n .joinLeftSemi(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val joined = {\n for {\n ab <- left if rightKeys.contains(ab.a)\n } yield ab\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= leftDs.schema\n\n (joined.sorted ?= joinedData) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n\n test(\"ab.joinRight(ac)(ab.a == ac.a)\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering,\n C : TypedEncoder : Ordering\n ](left: List[X2[A, B]], right: List[X2[A, C]]): Prop = {\n val leftDs = TypedDataset.create(left)\n val rightDs = TypedDataset.create(right)\n val joinedDs = leftDs\n .joinRight(rightDs)(leftDs.col('a) === rightDs.col('a))\n\n val joinedData = joinedDs.collect().run().toVector.sorted\n\n val leftKeys = left.map(_.a).toSet\n val joined = {\n for {\n ab <- left\n ac <- right if ac.a == ab.a\n } yield (Some(ab), ac)\n }.toVector ++ {\n for {\n ac <- right if !leftKeys.contains(ac.a)\n } yield (None, ac)\n }.toVector\n\n val equalSchemas = joinedDs.schema ?= StructType(Seq(\n StructField(\"_1\", leftDs.schema, nullable = true),\n StructField(\"_2\", rightDs.schema, nullable = false)))\n\n (joined.sorted ?= joinedData) && (joinedData.map(_._2).toSet ?= right.toSet) && equalSchemas\n }\n\n check(forAll(prop[Int, Long, String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/LitTests.scala", "content": "package frameless\n\nimport frameless.functions.lit\n\nimport org.scalatest.matchers.should.Matchers\n\nimport org.scalacheck.Prop, Prop._\n\nimport RecordEncoderTests.Name\n\nclass LitTests extends TypedDatasetSuite with Matchers {\n def prop[A: TypedEncoder](value: A)(implicit i0: shapeless.Refute[IsValueClass[A]]): Prop = {\n val df: TypedDataset[Int] = TypedDataset.create(1 :: Nil)\n\n val l: TypedColumn[Int, A] = lit(value)\n\n // filter forces whole codegen\n val elems = df.deserialized.filter((_:Int) => true).select(l)\n .collect()\n .run()\n .toVector\n\n // otherwise it uses local relation\n val localElems = df.select(l)\n .collect()\n .run()\n .toVector\n\n val expected = Vector(value)\n\n (localElems ?= expected) && (elems ?= expected)\n }\n\n test(\"select(lit(...))\") {\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n\n check(prop[Option[Int]] _)\n check(prop[Option[String]] _)\n\n check(prop[Vector[Long]] _)\n check(prop[Vector[X1[Long]]] _)\n\n check(prop[Vector[String]] _)\n check(prop[Vector[X1[String]]] _)\n\n check(prop[X1[Int]] _)\n check(prop[X1[X1[Int]]] _)\n\n check(prop[Food] _)\n\n // doesn't work, object has to be serializable\n // check(prop[frameless.LocalDateTime] _)\n }\n\n test(\"support value class\") {\n val initial = Seq(\n Q(name = new Name(\"Foo\"), id = 1),\n Q(name = new Name(\"Bar\"), id = 2))\n val ds = TypedDataset.create(initial)\n\n ds.collect.run() shouldBe initial\n\n val lorem = new Name(\"Lorem\")\n\n ds.withColumnReplaced('name, functions.litValue(lorem)).\n collect.run() shouldBe initial.map(_.copy(name = lorem))\n }\n\n test(\"support optional value class\") {\n val initial = Seq(\n R(name = \"Foo\", id = 1, alias = None),\n R(name = \"Bar\", id = 2, alias = Some(new Name(\"Lorem\"))))\n val ds = TypedDataset.create(initial)\n\n ds.collect.run() shouldBe initial\n\n val someIpsum: Option[Name] = Some(new Name(\"Ipsum\"))\n\n val lit = functions.litValue(someIpsum)\n val tds = ds.withColumnReplaced('alias, functions.litValue(someIpsum))\n\n tds.queryExecution.toString() should include (lit.toString)\n\n tds.\n collect.run() shouldBe initial.map(_.copy(alias = someIpsum))\n\n ds.withColumnReplaced('alias, functions.litValue(Option.empty[Name])).\n collect.run() shouldBe initial.map(_.copy(alias = None))\n }\n\n test(\"#205: comparing literals encoded using Injection\") {\n import org.apache.spark.sql.catalyst.util.DateTimeUtils\n implicit val dateAsInt: Injection[java.sql.Date, Int] =\n Injection(DateTimeUtils.fromJavaDate, DateTimeUtils.toJavaDate)\n\n val today = new java.sql.Date(System.currentTimeMillis)\n val data = Vector(P(42, today))\n val tds = TypedDataset.create(data)\n\n tds.filter(tds('d) === today).collect.run().map(_.i) shouldBe Seq(42)\n }\n}\n\nfinal case class P(i: Int, d: java.sql.Date)\n\nfinal case class Q(id: Int, name: Name)\n\nfinal case class R(id: Int, name: String, alias: Option[Name])\n" }, { "path": "dataset/src/test/scala/frameless/NumericTests.scala", "content": "package frameless\n\nimport org.apache.spark.sql.Encoder\nimport org.scalacheck.{Arbitrary, Gen, Prop}\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\n\nimport scala.reflect.ClassTag\n\nclass NumericTests extends TypedDatasetSuite with Matchers {\n test(\"plus\") {\n def prop[A: TypedEncoder: CatalystNumeric: Numeric](a: A, b: A): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[Numeric[A]].plus(a, b)\n val got = df.select(df.col('a) + df.col('b)).collect().run()\n\n got ?= (result :: Nil)\n }\n\n check(prop[BigDecimal] _)\n check(prop[Byte] _)\n check(prop[Double] _)\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[Short] _)\n }\n\n test(\"minus\") {\n def prop[A: TypedEncoder: CatalystNumeric: Numeric](a: A, b: A): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[Numeric[A]].minus(a, b)\n val got = df.select(df.col('a) - df.col('b)).collect().run()\n\n got ?= (result :: Nil)\n }\n\n check(prop[BigDecimal] _)\n check(prop[Byte] _)\n check(prop[Double] _)\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[Short] _)\n }\n\n test(\"multiply\") {\n def prop[A: TypedEncoder : CatalystNumeric : Numeric : ClassTag](a: A, b: A): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = implicitly[Numeric[A]].times(a, b)\n val got = df.select(df.col('a) * df.col('b)).collect().run()\n\n got ?= (result :: Nil)\n }\n\n check(prop[Byte] _)\n check(prop[Double] _)\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[Short] _)\n }\n\n test(\"divide\") {\n def prop[A: TypedEncoder: CatalystNumeric: Numeric](a: A, b: A)(implicit cd: CatalystDivisible[A, Double]): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n if (b == 0) proved else {\n val div: Double = implicitly[Numeric[A]].toDouble(a) / implicitly[Numeric[A]].toDouble(b)\n val got: Seq[Double] = df.select(df.col('a) / df.col('b)).collect().run()\n\n got ?= (div :: Nil)\n }\n }\n\n check(prop[Byte ] _)\n check(prop[Double] _)\n check(prop[Int ] _)\n check(prop[Long ] _)\n check(prop[Short ] _)\n }\n\n test(\"divide BigDecimals\") {\n def prop(a: BigDecimal, b: BigDecimal): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n if (b.doubleValue == 0) proved else {\n // Spark performs something in between Double division and BigDecimal division,\n // we approximate it using double vision and `approximatelyEqual`:\n val div = BigDecimal(a.doubleValue / b.doubleValue)\n val got = df.select(df.col('a) / df.col('b)).collect().run()\n approximatelyEqual(got.head, div)\n }\n }\n\n check(prop _)\n }\n\n test(\"multiply BigDecimal\") {\n def prop(a: BigDecimal, b: BigDecimal): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n val result = BigDecimal(a.doubleValue * b.doubleValue)\n val got = df.select(df.col('a) * df.col('b)).collect().run()\n approximatelyEqual(got.head, result)\n }\n\n check(prop _)\n }\n\n trait NumericMod[T] {\n def mod(a: T, b: T): T\n }\n\n object NumericMod {\n implicit val byteInstance = new NumericMod[Byte] {\n def mod(a: Byte, b: Byte) = (a % b).toByte\n }\n implicit val doubleInstance = new NumericMod[Double] {\n def mod(a: Double, b: Double) = a % b\n }\n implicit val floatInstance = new NumericMod[Float] {\n def mod(a: Float, b: Float) = a % b\n }\n implicit val intInstance = new NumericMod[Int] {\n def mod(a: Int, b: Int) = a % b\n }\n implicit val longInstance = new NumericMod[Long] {\n def mod(a: Long, b: Long) = a % b\n }\n implicit val shortInstance = new NumericMod[Short] {\n def mod(a: Short, b: Short) = (a % b).toShort\n }\n implicit val bigDecimalInstance = new NumericMod[BigDecimal] {\n def mod(a: BigDecimal, b: BigDecimal) = a % b\n }\n }\n\n test(\"mod\") {\n import NumericMod._\n\n def prop[A: TypedEncoder : CatalystNumeric : NumericMod](a: A, b: A): Prop = {\n val df = TypedDataset.create(X2(a, b) :: Nil)\n if (b == 0) proved else {\n val mod: A = implicitly[NumericMod[A]].mod(a, b)\n val got: Seq[A] = df.select(df.col('a) % df.col('b)).collect().run()\n\n got ?= (mod :: Nil)\n }\n }\n\n check(prop[Byte] _)\n check(prop[Double] _)\n check(prop[Int ] _)\n check(prop[Long ] _)\n check(prop[Short ] _)\n check(prop[BigDecimal] _)\n }\n\n test(\"a mod lit(b)\"){\n import NumericMod._\n\n def prop[A: TypedEncoder : CatalystNumeric : NumericMod](elem: A, data: X1[A]): Prop = {\n val dataset = TypedDataset.create(Seq(data))\n val a = dataset.col('a)\n if (elem == 0) proved else {\n val mod: A = implicitly[NumericMod[A]].mod(data.a, elem)\n val got: Seq[A] = dataset.select(a % elem).collect().run()\n\n got ?= (mod :: Nil)\n }\n }\n\n check(prop[Byte] _)\n check(prop[Double] _)\n check(prop[Int ] _)\n check(prop[Long ] _)\n check(prop[Short ] _)\n check(prop[BigDecimal] _)\n }\n\n test(\"isNaN\") {\n val spark = session\n import spark.implicits._\n\n implicit val doubleWithNaN = Arbitrary {\n implicitly[Arbitrary[Double]].arbitrary.flatMap(Gen.oneOf(_, Double.NaN))\n }\n implicit val x1 = Arbitrary{ doubleWithNaN.arbitrary.map(X1(_)) }\n\n def prop[A : TypedEncoder : Encoder : CatalystNaN](data: List[X1[A]]): Prop = {\n val ds = TypedDataset.create(data)\n\n val expected = ds.toDF().filter(!$\"a\".isNaN).map(_.getAs[A](0)).collect().toSeq\n val rs = ds.filter(!ds('a).isNaN).collect().run().map(_.a)\n\n rs ?= expected\n }\n\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"isNaN with non-nan types should not compile\") {\n val ds = TypedDataset.create((1, false, 'a, \"b\") :: Nil)\n\n \"ds.filter(ds('_1).isNaN)\" shouldNot typeCheck\n \"ds.filter(ds('_2).isNaN)\" shouldNot typeCheck\n \"ds.filter(ds('_3).isNaN)\" shouldNot typeCheck\n \"ds.filter(ds('_4).isNaN)\" shouldNot typeCheck\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/OrderByTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\nimport org.apache.spark.sql.Column\nimport org.scalatest.matchers.should.Matchers\n\nclass OrderByTests extends TypedDatasetSuite with Matchers {\n def sortings[A : CatalystOrdered, T]: Seq[(TypedColumn[T, A] => SortedTypedColumn[T, A], Column => Column)] = Seq(\n (_.desc, _.desc),\n (_.asc, _.asc),\n (t => t, t => t) //default ascending\n )\n\n test(\"single column non nullable orderBy\") {\n def prop[A: TypedEncoder : CatalystOrdered](data: Vector[X1[A]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X1[A]].map { case (typ, untyp) =>\n ds.dataset.orderBy(untyp(ds.dataset.col(\"a\"))).collect().toVector.?=(\n ds.orderBy(typ(ds('a))).collect().run().toVector)\n }.reduce(_ && _)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[String] _))\n }\n\n test(\"single column non nullable partition sorting\") {\n def prop[A: TypedEncoder : CatalystOrdered](data: Vector[X1[A]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X1[A]].map { case (typ, untyp) =>\n ds.dataset.sortWithinPartitions(untyp(ds.dataset.col(\"a\"))).collect().toVector.?=(\n ds.sortWithinPartitions(typ(ds('a))).collect().run().toVector)\n }.reduce(_ && _)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Boolean] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Float] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[SQLDate] _))\n check(forAll(prop[SQLTimestamp] _))\n check(forAll(prop[String] _))\n }\n\n test(\"two columns non nullable orderBy\") {\n def prop[A: TypedEncoder : CatalystOrdered, B: TypedEncoder : CatalystOrdered](data: Vector[X2[A,B]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X2[A, B]].reverse.zip(sortings[B, X2[A, B]]).map { case ((typA, untypA), (typB, untypB)) =>\n val vanillaSpark = ds.dataset.orderBy(untypA(ds.dataset.col(\"a\")), untypB(ds.dataset.col(\"b\"))).collect().toVector\n vanillaSpark.?=(ds.orderBy(typA(ds('a)), typB(ds('b))).collect().run().toVector).&&(\n vanillaSpark ?= ds.orderByMany(typA(ds('a)), typB(ds('b))).collect().run().toVector\n )\n }.reduce(_ && _)\n }\n\n check(forAll(prop[SQLDate, Long] _))\n check(forAll(prop[String, Boolean] _))\n check(forAll(prop[SQLTimestamp, Long] _))\n }\n\n test(\"two columns non nullable partition sorting\") {\n def prop[A: TypedEncoder : CatalystOrdered, B: TypedEncoder : CatalystOrdered](data: Vector[X2[A,B]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X2[A, B]].reverse.zip(sortings[B, X2[A, B]]).map { case ((typA, untypA), (typB, untypB)) =>\n val vanillaSpark = ds.dataset.sortWithinPartitions(untypA(ds.dataset.col(\"a\")), untypB(ds.dataset.col(\"b\"))).collect().toVector\n vanillaSpark.?=(ds.sortWithinPartitions(typA(ds('a)), typB(ds('b))).collect().run().toVector).&&(\n vanillaSpark ?= ds.sortWithinPartitionsMany(typA(ds('a)), typB(ds('b))).collect().run().toVector\n )\n }.reduce(_ && _)\n }\n\n check(forAll(prop[SQLDate, Long] _))\n check(forAll(prop[String, Boolean] _))\n check(forAll(prop[SQLTimestamp, Long] _))\n }\n\n test(\"three columns non nullable orderBy\") {\n def prop[A: TypedEncoder : CatalystOrdered, B: TypedEncoder : CatalystOrdered](data: Vector[X3[A,B,A]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X3[A, B, A]].reverse\n .zip(sortings[B, X3[A, B, A]])\n .zip(sortings[A, X3[A, B, A]])\n .map { case (((typA, untypA), (typB, untypB)), (typA2, untypA2)) =>\n val vanillaSpark = ds.dataset\n .orderBy(untypA(ds.dataset.col(\"a\")), untypB(ds.dataset.col(\"b\")), untypA2(ds.dataset.col(\"c\")))\n .collect().toVector\n\n vanillaSpark.?=(ds.orderBy(typA(ds('a)), typB(ds('b)), typA2(ds('c))).collect().run().toVector).&&(\n vanillaSpark ?= ds.orderByMany(typA(ds('a)), typB(ds('b)), typA2(ds('c))).collect().run().toVector\n )\n }.reduce(_ && _)\n }\n\n check(forAll(prop[SQLDate, Long] _))\n check(forAll(prop[String, Boolean] _))\n check(forAll(prop[SQLTimestamp, Long] _))\n }\n\n test(\"three columns non nullable partition sorting\") {\n def prop[A: TypedEncoder : CatalystOrdered, B: TypedEncoder : CatalystOrdered](data: Vector[X3[A,B,A]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[A, X3[A, B, A]].reverse\n .zip(sortings[B, X3[A, B, A]])\n .zip(sortings[A, X3[A, B, A]])\n .map { case (((typA, untypA), (typB, untypB)), (typA2, untypA2)) =>\n val vanillaSpark = ds.dataset\n .sortWithinPartitions(untypA(ds.dataset.col(\"a\")), untypB(ds.dataset.col(\"b\")), untypA2(ds.dataset.col(\"c\")))\n .collect().toVector\n\n vanillaSpark.?=(ds.sortWithinPartitions(typA(ds('a)), typB(ds('b)), typA2(ds('c))).collect().run().toVector).&&(\n vanillaSpark ?= ds.sortWithinPartitionsMany(typA(ds('a)), typB(ds('b)), typA2(ds('c))).collect().run().toVector\n )\n }.reduce(_ && _)\n }\n\n check(forAll(prop[SQLDate, Long] _))\n check(forAll(prop[String, Boolean] _))\n check(forAll(prop[SQLTimestamp, Long] _))\n }\n\n test(\"sort support for mixed default and explicit ordering\") {\n def prop[A: TypedEncoder : CatalystOrdered, B: TypedEncoder : CatalystOrdered](data: Vector[X2[A, B]]): Prop = {\n val ds = TypedDataset.create(data)\n\n ds.dataset.orderBy(ds.dataset.col(\"a\"), ds.dataset.col(\"b\").desc).collect().toVector.?=(\n ds.orderByMany(ds('a), ds('b).desc).collect().run().toVector) &&\n ds.dataset.sortWithinPartitions(ds.dataset.col(\"a\"), ds.dataset.col(\"b\").desc).collect().toVector.?=(\n ds.sortWithinPartitionsMany(ds('a), ds('b).desc).collect().run().toVector)\n }\n\n check(forAll(prop[SQLDate, Long] _))\n check(forAll(prop[String, Boolean] _))\n check(forAll(prop[SQLTimestamp, Long] _))\n }\n\n test(\"fail when selected column is not sortable\") {\n val d = TypedDataset.create(X2(1, Map(1 -> 2)) :: X2(2, Map(2 -> 2)) :: Nil)\n d.orderBy(d('a).desc)\n illTyped(\"\"\"d.orderBy(d('b).desc)\"\"\")\n illTyped(\"\"\"d.sortWithinPartitions(d('b).desc)\"\"\")\n }\n\n test(\"derives a CatalystOrdered for case classes when all fields are comparable\") {\n type T[A, B] = X3[Int, Boolean, X2[A, B]]\n def prop[\n A: TypedEncoder : CatalystOrdered,\n B: TypedEncoder : CatalystOrdered\n ](data: Vector[T[A, B]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[X2[A, B], T[A, B]].map { case (typX2, untypX2) =>\n val vanilla = ds.dataset.orderBy(untypX2(ds.dataset.col(\"c\"))).collect().toVector\n val frameless = ds.orderBy(typX2(ds('c))).collect().run.toVector\n vanilla ?= frameless\n }.reduce(_ && _)\n }\n\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[(String, SQLDate), Float] _))\n // Check that nested case classes are properly derived too\n check(forAll(prop[X2[Boolean, Float], X4[SQLTimestamp, Double, Short, Byte]] _))\n }\n\n test(\"derives a CatalystOrdered for tuples when all fields are comparable\") {\n type T[A, B] = X2[Int, (A, B)]\n def prop[\n A: TypedEncoder : CatalystOrdered,\n B: TypedEncoder : CatalystOrdered\n ](data: Vector[T[A, B]]): Prop = {\n val ds = TypedDataset.create(data)\n\n sortings[(A, B), T[A, B]].map { case (typX2, untypX2) =>\n val vanilla = ds.dataset.orderBy(untypX2(ds.dataset.col(\"b\"))).collect().toVector\n val frameless = ds.orderBy(typX2(ds('b))).collect().run.toVector\n vanilla ?= frameless\n }.reduce(_ && _)\n }\n\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[(String, SQLDate), Float] _))\n check(forAll(prop[X2[Boolean, Float], X1[(SQLTimestamp, Double, Short, Byte)]] _))\n }\n\n test(\"fails to compile when one of the field isn't comparable\") {\n type T = X2[Int, X2[Int, Map[String, String]]]\n val d = TypedDataset.create(X2(1, X2(2, Map(\"not\" -> \"comparable\"))) :: Nil)\n illTyped(\"d.orderBy(d('b).desc)\", \"\"\"Cannot compare columns of type frameless.X2\\[Int,scala.collection.immutable.Map\\[String,String]].\"\"\")\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/RecordEncoderTests.scala", "content": "package frameless\n\nimport org.apache.spark.sql.{Row, functions => F}\nimport org.apache.spark.sql.types.{\n ArrayType,\n BinaryType,\n DecimalType,\n IntegerType,\n LongType,\n MapType,\n ObjectType,\n StringType,\n StructField,\n StructType\n}\n\nimport shapeless.{HList, LabelledGeneric}\nimport shapeless.test.illTyped\n\nimport org.scalatest.matchers.should.Matchers\n\nfinal class RecordEncoderTests extends TypedDatasetSuite with Matchers {\n test(\"Unable to encode products made from units only\") {\n illTyped(\"TypedEncoder[UnitsOnly]\")\n }\n\n test(\"Dropping fields\") {\n def dropUnitValues[L <: HList](l: L)(implicit d: DropUnitValues[L]): d.Out = d(l)\n val fields = LabelledGeneric[TupleWithUnits].to(TupleWithUnits(42, \"something\"))\n dropUnitValues(fields) shouldEqual LabelledGeneric[(Int, String)].to((42, \"something\"))\n }\n\n test(\"Representation skips units\") {\n assert(TypedEncoder[(Int, String)].catalystRepr == TypedEncoder[TupleWithUnits].catalystRepr)\n }\n\n test(\"Serialization skips units\") {\n val df = session.createDataFrame(Seq((1, \"one\"), (2, \"two\")))\n val ds = df.as[TupleWithUnits](TypedExpressionEncoder[TupleWithUnits])\n val tds = TypedDataset.create(Seq(TupleWithUnits(1, \"one\"), TupleWithUnits(2, \"two\")))\n\n df.collect shouldEqual tds.toDF.collect\n ds.collect.toSeq shouldEqual tds.collect.run\n }\n\n test(\"Empty nested record value becomes null on serialization\") {\n val ds = TypedDataset.create(Seq(OptionalNesting(Option.empty)))\n val df = ds.toDF\n df.na.drop.count shouldBe 0\n }\n\n test(\"Empty nested record value becomes none on deserialization\") {\n val rdd = sc.parallelize(Seq(Row(null)))\n val schema = TypedEncoder[OptionalNesting].catalystRepr.asInstanceOf[StructType]\n val df = session.createDataFrame(rdd, schema)\n val ds = TypedDataset.createUnsafe(df)(TypedEncoder[OptionalNesting])\n\n ds.firstOption.run.get.o.isEmpty shouldBe true\n }\n\n test(\"Deeply nested optional values have correct deserialization\") {\n val rdd = sc.parallelize(Seq(Row(true, Row(null, null))))\n type NestedOptionPair = X2[Boolean, Option[X2[Option[Int], Option[String]]]]\n val schema = TypedEncoder[NestedOptionPair].catalystRepr.asInstanceOf[StructType]\n val df = session.createDataFrame(rdd, schema)\n val ds = TypedDataset.createUnsafe(df)(TypedEncoder[NestedOptionPair])\n ds.firstOption.run.get shouldBe X2(true, Some(X2(None, None)))\n }\n\n test(\"Nesting with Seq\") {\n import RecordEncoderTests._\n\n val obj = C(B(Seq(A(1))))\n val rdd = sc.parallelize(Seq(obj))\n val ds = session.createDataset(rdd)(TypedExpressionEncoder[C])\n\n ds.collect.head shouldBe obj\n }\n\n test(\"Nesting with Set\") {\n import RecordEncoderTests._\n\n val obj = E(Set(B(Seq(A(1)))))\n val rdd = sc.parallelize(Seq(obj))\n val ds = session.createDataset(rdd)(TypedExpressionEncoder[E])\n\n ds.collect.head shouldBe obj\n }\n\n test(\"Scalar value class\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[Name]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[Name])\n\n encoder.catalystRepr shouldBe StructType(\n Seq(StructField(\"value\", StringType, false)))\n\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n TypedDataset\n .createUnsafe[Name](Seq(\"Foo\", \"Bar\").toDF)(encoder)\n .collect().run() shouldBe Seq(new Name(\"Foo\"), new Name(\"Bar\"))\n\n }\n\n test(\"Case class with value class field\") {\n import RecordEncoderTests._\n\n illTyped(\n // As `Person` is not a Value class\n \"val _: RecordFieldEncoder[Person] = RecordFieldEncoder.valueClass\")\n\n val fieldEncoder: RecordFieldEncoder[Name] = RecordFieldEncoder.valueClass\n\n fieldEncoder.encoder.catalystRepr shouldBe StringType\n fieldEncoder.encoder.jvmRepr shouldBe ObjectType(classOf[String])\n\n // Encode as a Person field\n val encoder = TypedEncoder[Person]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[Person])\n\n val expectedPersonStructType = StructType(Seq(\n StructField(\"name\", StringType, false),\n StructField(\"age\", IntegerType, false)))\n\n encoder.catalystRepr shouldBe expectedPersonStructType\n\n val unsafeDs: TypedDataset[Person] = {\n val rdd = sc.parallelize(Seq(\n Row.fromTuple(\"Foo\" -> 2),\n Row.fromTuple(\"Bar\" -> 3)\n ))\n val df = session.createDataFrame(rdd, expectedPersonStructType)\n\n TypedDataset.createUnsafe(df)(encoder)\n }\n\n val expected = Seq(\n Person(new Name(\"Foo\"), 2), Person(new Name(\"Bar\"), 3))\n\n unsafeDs.collect.run() shouldBe expected\n\n // Safely created DS\n val safeDs = TypedDataset.create(expected)\n\n safeDs.collect.run() shouldBe expected\n\n val lorem = new Name(\"Lorem\")\n\n safeDs.withColumnReplaced('name, functions.litValue(lorem)).\n collect.run() shouldBe expected.map(_.copy(name = lorem))\n }\n\n test(\"Case class with value class as optional field\") {\n import RecordEncoderTests._\n\n illTyped( // As `Person` is not a Value class\n \"\"\"val _: RecordFieldEncoder[Option[Person]] =\n RecordFieldEncoder.optionValueClass\"\"\")\n\n val fieldEncoder: RecordFieldEncoder[Option[Name]] =\n RecordFieldEncoder.optionValueClass\n\n fieldEncoder.encoder.catalystRepr shouldBe StringType\n\n fieldEncoder.encoder. // !StringType\n jvmRepr shouldBe ObjectType(classOf[Option[_]])\n\n // Encode as a Person field\n val encoder = TypedEncoder[User]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[User])\n\n val expectedPersonStructType = StructType(Seq(\n StructField(\"id\", LongType, false),\n StructField(\"name\", StringType, true)))\n\n encoder.catalystRepr shouldBe expectedPersonStructType\n\n val ds1: TypedDataset[User] = {\n val rdd = sc.parallelize(Seq(\n Row(1L, null),\n Row(2L, \"Foo\")\n ))\n\n val df = session.createDataFrame(rdd, expectedPersonStructType)\n\n TypedDataset.createUnsafe(df)(encoder)\n }\n\n ds1.collect.run() shouldBe Seq(\n User(1L, None),\n User(2L, Some(new Name(\"Foo\"))))\n\n val ds2: TypedDataset[User] = {\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val df1 = Seq(\n \"\"\"{\"id\":3,\"label\":\"unused\"}\"\"\",\n \"\"\"{\"id\":4,\"name\":\"Lorem\"}\"\"\",\n \"\"\"{\"id\":5,\"name\":null}\"\"\"\n ).toDF\n\n val df2 = df1.withColumn(\n \"jsonValue\",\n F.from_json(df1.col(\"value\"), expectedPersonStructType)).\n select(\"jsonValue.id\", \"jsonValue.name\")\n\n TypedDataset.createUnsafe[User](df2)\n }\n\n val expected = Seq(\n User(3L, None),\n User(4L, Some(new Name(\"Lorem\"))),\n User(5L, None))\n\n ds2.collect.run() shouldBe expected\n\n // Safely created ds\n TypedDataset.create(expected).collect.run() shouldBe expected\n }\n\n test(\"Case class with simple Map\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[D]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[D])\n\n val expectedStructType = StructType(Seq(\n StructField(\"m\", MapType(\n keyType = StringType,\n valueType = IntegerType,\n valueContainsNull = false), false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val ds1 = TypedDataset.createUnsafe[D] {\n val df = Seq(\n \"\"\"{\"m\":{\"pizza\":1,\"sushi\":2}}\"\"\",\n \"\"\"{\"m\":{\"red\":3,\"blue\":4}}\"\"\",\n ).toDF\n\n df.withColumn(\n \"jsonValue\",\n F.from_json(df.col(\"value\"), expectedStructType)).\n select(\"jsonValue.*\")\n }\n\n val expected = Seq(\n D(m = Map(\"pizza\" -> 1, \"sushi\" -> 2)),\n D(m = Map(\"red\" -> 3, \"blue\" -> 4)))\n\n ds1.collect.run() shouldBe expected\n\n val m2 = Map(\"updated\" -> 5)\n\n val ds2 = ds1.withColumnReplaced('m, functions.lit(m2))\n\n ds2.collect.run() shouldBe expected.map(_.copy(m = m2))\n }\n\n test(\"Case class with Map & Value class\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[Student]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[Student])\n\n val expectedStudentStructType = StructType(Seq(\n StructField(\"name\", StringType, false),\n StructField(\"grades\", MapType(\n keyType = StringType,\n valueType = DecimalType.SYSTEM_DEFAULT,\n valueContainsNull = false), false)))\n\n encoder.catalystRepr shouldBe expectedStudentStructType\n\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val ds1 = TypedDataset.createUnsafe[Student] {\n val df = Seq(\n \"\"\"{\"name\":\"Foo\",\"grades\":{\"math\":1,\"physics\":\"23.4\"}}\"\"\",\n \"\"\"{\"name\":\"Bar\",\"grades\":{\"biology\":18.5,\"geography\":4}}\"\"\",\n ).toDF\n\n df.withColumn(\n \"jsonValue\",\n F.from_json(df.col(\"value\"), expectedStudentStructType)).\n select(\"jsonValue.*\")\n }\n\n val expected = Seq(\n Student(name = \"Foo\", grades = Map(\n new Subject(\"math\") -> new Grade(BigDecimal(1)),\n new Subject(\"physics\") -> new Grade(BigDecimal(23.4D)))),\n Student(name = \"Bar\", grades = Map(\n new Subject(\"biology\") -> new Grade(BigDecimal(18.5)),\n new Subject(\"geography\") -> new Grade(BigDecimal(4L)))))\n\n ds1.collect.run() shouldBe expected\n\n val grades = Map[Subject, Grade](\n new Subject(\"any\") -> new Grade(BigDecimal(Long.MaxValue) + 1L))\n\n val ds2 = ds1.withColumnReplaced('grades, functions.lit(grades))\n\n ds2.collect.run() shouldBe Seq(\n Student(\"Foo\", grades), Student(\"Bar\", grades))\n }\n\n test(\"Encode binary array\") {\n val encoder = TypedEncoder[Tuple2[String, Array[Byte]]]\n\n encoder.jvmRepr shouldBe ObjectType(\n classOf[Tuple2[String, Array[Byte]]])\n\n val expectedStructType = StructType(Seq(\n StructField(\"_1\", StringType, false),\n StructField(\"_2\", BinaryType, false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val ds1: TypedDataset[(String, Array[Byte])] = {\n val rdd = sc.parallelize(Seq(\n Row.fromTuple(\"Foo\" -> Array[Byte](3, 4)),\n Row.fromTuple(\"Bar\" -> Array[Byte](5))\n ))\n val df = session.createDataFrame(rdd, expectedStructType)\n\n TypedDataset.createUnsafe(df)(encoder)\n }\n\n val expected = Seq(\"Foo\" -> Seq[Byte](3, 4), \"Bar\" -> Seq[Byte](5))\n\n ds1.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected\n\n val subjects = \"lorem\".getBytes(\"UTF-8\").toSeq\n\n val ds2 = ds1.withColumnReplaced('_2, functions.lit(subjects.toArray))\n\n ds2.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected.map(_.copy(_2 = subjects))\n }\n\n test(\"Encode simple array\") {\n val encoder = TypedEncoder[Tuple2[String, Array[Int]]]\n\n encoder.jvmRepr shouldBe ObjectType(\n classOf[Tuple2[String, Array[Int]]])\n\n val expectedStructType = StructType(Seq(\n StructField(\"_1\", StringType, false),\n StructField(\"_2\", ArrayType(IntegerType, false), false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val ds1 = TypedDataset.createUnsafe[(String, Array[Int])] {\n val df = Seq(\n \"\"\"{\"_1\":\"Foo\", \"_2\":[3, 4]}\"\"\",\n \"\"\"{\"_1\":\"Bar\", \"_2\":[5]}\"\"\",\n ).toDF\n\n df.withColumn(\n \"jsonValue\",\n F.from_json(df.col(\"value\"), expectedStructType)).\n select(\"jsonValue.*\")\n }\n\n val expected = Seq(\"Foo\" -> Seq(3, 4), \"Bar\" -> Seq(5))\n\n ds1.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected\n\n val subjects = Seq(6, 6, 7)\n\n val ds2 = ds1.withColumnReplaced('_2, functions.lit(subjects.toArray))\n\n ds2.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected.map(_.copy(_2 = subjects))\n }\n\n test(\"Encode array of Value class\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[Tuple2[String, Array[Subject]]]\n\n encoder.jvmRepr shouldBe ObjectType(\n classOf[Tuple2[String, Array[Subject]]])\n\n val expectedStructType = StructType(Seq(\n StructField(\"_1\", StringType, false),\n StructField(\"_2\", ArrayType(StringType, false), false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val ds1 = TypedDataset.createUnsafe[(String, Array[Subject])] {\n val df = Seq(\n \"\"\"{\"_1\":\"Foo\", \"_2\":[\"math\",\"physics\"]}\"\"\",\n \"\"\"{\"_1\":\"Bar\", \"_2\":[\"biology\",\"geography\"]}\"\"\",\n ).toDF\n\n df.withColumn(\n \"jsonValue\",\n F.from_json(df.col(\"value\"), expectedStructType)).\n select(\"jsonValue.*\")\n }\n\n val expected = Seq(\n \"Foo\" -> Seq(new Subject(\"math\"), new Subject(\"physics\")),\n \"Bar\" -> Seq(new Subject(\"biology\"), new Subject(\"geography\")))\n\n ds1.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected\n\n val subjects = Seq(new Subject(\"lorem\"), new Subject(\"ipsum\"))\n\n val ds2 = ds1.withColumnReplaced('_2, functions.lit(subjects.toArray))\n\n ds2.collect.run().map {\n case (_1, _2) => _1 -> _2.toSeq\n } shouldBe expected.map(_.copy(_2 = subjects))\n }\n\n test(\"Encode case class with simple Seq\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[B]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[B])\n\n val expectedStructType = StructType(Seq(\n StructField(\"a\", ArrayType(StructType(Seq(\n StructField(\"x\", IntegerType, false))), false), false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val ds1: TypedDataset[B] = {\n val rdd = sc.parallelize(Seq(\n Row.fromTuple(Tuple1(Seq(\n Row.fromTuple(Tuple1[Int](1)),\n Row.fromTuple(Tuple1[Int](3))\n ))),\n Row.fromTuple(Tuple1(Seq(\n Row.fromTuple(Tuple1[Int](2))\n )))\n ))\n val df = session.createDataFrame(rdd, expectedStructType)\n\n TypedDataset.createUnsafe(df)(encoder)\n }\n\n val expected = Seq(B(Seq(A(1), A(3))), B(Seq(A(2))))\n\n ds1.collect.run() shouldBe expected\n\n val as = Seq(A(5), A(6))\n\n val ds2 = ds1.withColumnReplaced('a, functions.lit(as))\n\n ds2.collect.run() shouldBe expected.map(_.copy(a = as))\n }\n\n test(\"Encode case class with Value class\") {\n import RecordEncoderTests._\n\n val encoder = TypedEncoder[Tuple2[Int, Seq[Name]]]\n\n encoder.jvmRepr shouldBe ObjectType(classOf[Tuple2[Int, Seq[Name]]])\n\n val expectedStructType = StructType(Seq(\n StructField(\"_1\", IntegerType, false),\n StructField(\"_2\", ArrayType(StringType, false), false)))\n\n encoder.catalystRepr shouldBe expectedStructType\n\n val ds1 = TypedDataset.createUnsafe[(Int, Seq[Name])] {\n val sqlContext = session.sqlContext\n import sqlContext.implicits._\n\n val df = Seq(\n \"\"\"{\"_1\":1, \"_2\":[\"foo\", \"bar\"]}\"\"\",\n \"\"\"{\"_1\":2, \"_2\":[\"lorem\"]}\"\"\",\n ).toDF\n\n df.withColumn(\n \"jsonValue\",\n F.from_json(df.col(\"value\"), expectedStructType)).\n select(\"jsonValue.*\")\n }\n\n val expected = Seq(\n 1 -> Seq(new Name(\"foo\"), new Name(\"bar\")),\n 2 -> Seq(new Name(\"lorem\")))\n\n ds1.collect.run() shouldBe expected\n }\n}\n\n// ---\n\ncase class UnitsOnly(a: Unit, b: Unit)\n\ncase class TupleWithUnits(\n u0: Unit, _1: Int, u1: Unit, u2: Unit, _2: String, u3: Unit)\n\nobject TupleWithUnits {\n def apply(_1: Int, _2: String): TupleWithUnits =\n TupleWithUnits((), _1, (), (), _2, ())\n}\n\ncase class OptionalNesting(o: Option[TupleWithUnits])\n\nobject RecordEncoderTests {\n case class A(x: Int)\n case class B(a: Seq[A])\n case class C(b: B)\n\n class Name(val value: String) extends AnyVal with Serializable {\n override def toString = s\"Name($value)\"\n }\n\n case class Person(name: Name, age: Int)\n\n case class User(id: Long, name: Option[Name])\n\n case class D(m: Map[String, Int])\n case class E(b: Set[B])\n\n final class Subject(val name: String) extends AnyVal with Serializable\n\n final class Grade(val value: BigDecimal) extends AnyVal with Serializable\n\n case class Student(name: String, grades: Map[Subject, Grade])\n}\n" }, { "path": "dataset/src/test/scala/frameless/SchemaTests.scala", "content": "package frameless\n\nimport frameless.functions.aggregate._\nimport frameless.functions._\nimport org.apache.spark.sql.types.StructType\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\n\nclass SchemaTests extends TypedDatasetSuite with Matchers {\n\n def structToNonNullable(struct: StructType): StructType = {\n StructType(struct.fields.map( f => f.copy(nullable = false)))\n }\n\n def prop[A](dataset: TypedDataset[A], ignoreNullable: Boolean = false): Prop = {\n val schema = dataset.dataset.schema\n\n Prop.all(\n if (!ignoreNullable)\n dataset.schema ?= schema\n else\n structToNonNullable(dataset.schema) ?= structToNonNullable(schema),\n if (!ignoreNullable)\n TypedExpressionEncoder.targetStructType(dataset.encoder) ?= schema\n else\n structToNonNullable(TypedExpressionEncoder.targetStructType(dataset.encoder)) ?= structToNonNullable(schema)\n )\n }\n\n test(\"schema of groupBy('a).agg(sum('b))\") {\n val df0 = TypedDataset.create(X2(1L, 1L) :: Nil)\n val _a = df0.col('a)\n val _b = df0.col('b)\n\n val df = df0.groupBy(_a).agg(sum(_b))\n\n check(prop(df, true))\n }\n\n test(\"schema of select(lit(1L))\") {\n val df0 = TypedDataset.create(\"test\" :: Nil)\n val df = df0.select(lit(1L))\n\n check(prop(df))\n }\n\n test(\"schema of select(lit(1L), lit(2L)).as[X2[Long, Long]]\") {\n val df0 = TypedDataset.create(\"test\" :: Nil)\n val df = df0.select(lit(1L), lit(2L)).as[X2[Long, Long]]\n\n check(prop(df))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/SelectTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\nimport scala.reflect.ClassTag\n\nclass SelectTests extends TypedDatasetSuite {\n test(\"select('a) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val dataset2 = dataset.select(A).collect().run().toVector\n val data2 = data.map { case X4(a, _, _, _) => a }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[X2[Int, Int], Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[UdtEncodedClass, Int, Int, Int] _))\n }\n\n test(\"select('a, 'b) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n eab: TypedEncoder[(A, B)],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val dataset2 = dataset.select(A, B).collect().run().toVector\n val data2 = data.map { case X4(a, b, _, _) => (a, b) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, String, Int, Int] _))\n }\n\n test(\"select('a, 'b, 'c) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n eab: TypedEncoder[(A, B, C)],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val dataset2 = dataset.select(A, B, C).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, _) => (a, b, c) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, String, Int, Int] _))\n }\n\n test(\"select('a,'b,'c,'d) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a, 'c) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1, a3).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a, c) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a,'c,'b) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1, a3, a2).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a, c, b) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a,'c,'b, 'a) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1, a3, a2, a1).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a, c, b, a) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a,'c,'b,'a,'c) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1, a3, a2, a1, a3).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a, c, b, a, c) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a,'b,'c,'d,'a,'c,'b,'a,'c, 'd) FROM abcd\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n eb: TypedEncoder[B],\n ec: TypedEncoder[C],\n ed: TypedEncoder[D],\n ex4: TypedEncoder[X4[A, B, C, D]],\n ca: ClassTag[A]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val a1 = dataset.col[A]('a)\n val a2 = dataset.col[B]('b)\n val a3 = dataset.col[C]('c)\n val a4 = dataset.col[D]('d)\n\n val dataset2 = dataset.select(a1, a2, a3, a4, a1, a3, a2, a1, a3, a4).collect().run().toVector\n val data2 = data.map { case X4(a, b, c, d) => (a, b, c, d, a, c, b, a, c, d) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[String, Boolean, Int, Float] _))\n }\n\n test(\"select('a.b)\") {\n def prop[A, B, C](data: Vector[X2[X2[A, B], C]])(\n implicit\n eabc: TypedEncoder[X2[X2[A, B], C]],\n eb: TypedEncoder[B],\n cb: ClassTag[B]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val AB = dataset.colMany('a, 'b)\n\n val dataset2 = dataset.select(AB).collect().run().toVector\n val data2 = data.map { case X2(X2(_, b), _) => b }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int, String, Double] _))\n }\n\n test(\"select with column expression addition\") {\n def prop[A](data: Vector[X1[A]], const: A)(\n implicit\n eabc: TypedEncoder[X1[A]],\n anum: CatalystNumeric[A],\n num: Numeric[A],\n eb: TypedEncoder[A]\n ): Prop = {\n val ds = TypedDataset.create(data)\n\n val dataset2 = ds.select(ds('a) + const).collect().run().toVector\n val data2 = data.map { case X1(a) => num.plus(a, const) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Short] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"select with column expression multiplication\") {\n def prop[A](data: Vector[X1[A]], const: A)(\n implicit\n eabc: TypedEncoder[X1[A]],\n anum: CatalystNumeric[A],\n num: Numeric[A],\n eb: TypedEncoder[A]\n ): Prop = {\n val ds = TypedDataset.create(data)\n\n val dataset2 = ds.select(ds('a) * const).collect().run().toVector\n val data2 = data.map { case X1(a) => num.times(a, const) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Short] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"select with column expression subtraction\") {\n def prop[A](data: Vector[X1[A]], const: A)(\n implicit\n eabc: TypedEncoder[X1[A]],\n cnum: CatalystNumeric[A],\n num: Numeric[A],\n eb: TypedEncoder[A]\n ): Prop = {\n val ds = TypedDataset.create(data)\n\n val dataset2 = ds.select(ds('a) - const).collect().run().toVector\n val data2 = data.map { case X1(a) => num.minus(a, const) }\n\n dataset2 ?= data2\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"select with column expression division\") {\n def prop[A](data: Vector[X1[A]], const: A)(\n implicit\n eabc: TypedEncoder[X1[A]],\n anum: CatalystNumeric[A],\n frac: Fractional[A],\n eb: TypedEncoder[A]\n ): Prop = {\n val ds = TypedDataset.create(data)\n\n if (const != 0) {\n val dataset2 = ds.select(ds('a) / const).collect().run().toVector.asInstanceOf[Vector[A]]\n val data2 = data.map { case X1(a) => frac.div(a, const) }\n dataset2 ?= data2\n } else 0 ?= 0\n }\n\n check(forAll(prop[Double] _))\n }\n\n test(\"tests to cover problematic dataframe column names during projections\") {\n case class Foo(i: Int)\n val e = TypedDataset.create[Foo](Foo(1) :: Nil)\n val t: TypedDataset[(Int, Int)] = e.select(e.col('i) * 2, e.col('i))\n assert(t.select(t.col('_1)).collect().run().toList === List(2))\n // Issue #54\n val fooT = t.select(t.col('_1)).deserialized.map(x => Tuple1.apply(x)).as[Foo]\n assert(fooT.select(fooT('i)).collect().run().toList === List(2))\n }\n\n test(\"unary - on arithmetic\") {\n val e = TypedDataset.create[(Int, String, Int)]((1, \"a\", 2) :: (2, \"b\", 4) :: (2, \"b\", 1) :: Nil)\n assert(e.select(-e('_1)).collect().run().toVector === Vector(-1, -2, -2))\n assert(e.select(-(e('_1) + e('_3))).collect().run().toVector === Vector(-3, -6, -3))\n }\n\n test(\"unary - on strings should not type check\") {\n val e = TypedDataset.create[(Int, String, Long)]((1, \"a\", 2L) :: (2, \"b\", 4L) :: (2, \"b\", 1L) :: Nil)\n illTyped(\"\"\"e.select( -e('_2) )\"\"\")\n }\n\n test(\"select with aggregation operations is not supported\") {\n val e = TypedDataset.create[(Int, String, Long)]((1, \"a\", 2L) :: (2, \"b\", 4L) :: (2, \"b\", 1L) :: Nil)\n illTyped(\"\"\"e.select(frameless.functions.aggregate.sum(e('_1)))\"\"\")\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/SelfJoinTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport org.apache.spark.sql.{SparkSession, functions => sparkFunctions}\n\nclass SelfJoinTests extends TypedDatasetSuite {\n // Without crossJoin.enabled=true Spark doesn't like trivial join conditions:\n // [error] Join condition is missing or trivial.\n // [error] Use the CROSS JOIN syntax to allow cartesian products between these relations.\n def allowTrivialJoin[T](body: => T)(implicit session: SparkSession): T = {\n val crossJoin = \"spark.sql.crossJoin.enabled\"\n val oldSetting = session.conf.get(crossJoin)\n session.conf.set(crossJoin, \"true\")\n val result = body\n session.conf.set(crossJoin, oldSetting)\n result\n }\n\n def allowAmbiguousJoin[T](body: => T)(implicit session: SparkSession): T = {\n val crossJoin = \"spark.sql.analyzer.failAmbiguousSelfJoin\"\n val oldSetting = session.conf.get(crossJoin)\n session.conf.set(crossJoin, \"false\")\n val result = body\n session.conf.set(crossJoin, oldSetting)\n result\n }\n\n test(\"self join with colLeft/colRight disambiguation\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering\n ](dx: List[X2[A, B]], d: X2[A, B]): Prop = allowAmbiguousJoin {\n val data = d :: dx\n val ds = TypedDataset.create(data)\n\n // This is the way to write unambiguous self-join in vanilla, see https://goo.gl/XnkSUD\n val df1 = ds.dataset.as(\"df1\")\n val df2 = ds.dataset.as(\"df2\")\n val vanilla = df1.join(df2,\n sparkFunctions.col(\"df1.a\") === sparkFunctions.col(\"df2.a\")).count()\n\n val typed = ds.joinInner(ds)(\n ds.colLeft('a) === ds.colRight('a)\n ).count().run()\n\n vanilla ?= typed\n }\n\n check(prop[Int, Int] _)\n }\n\n test(\"trivial self join\") {\n def prop[\n A : TypedEncoder : Ordering,\n B : TypedEncoder : Ordering\n ](dx: List[X2[A, B]], d: X2[A, B]): Prop =\n allowTrivialJoin { allowAmbiguousJoin {\n\n val data = d :: dx\n val ds = TypedDataset.create(data)\n val untyped = ds.dataset\n // Interestingly, even with aliasing it seems that it's impossible to\n // obtain a trivial join condition of shape df1.a == df1.a, Spark we\n // always interpret that as df1.a == df2.a. For the purpose of this\n // test we fall-back to lit(true) instead.\n // val trivial = sparkFunctions.col(\"df1.a\") === sparkFunctions.col(\"df1.a\")\n val trivial = sparkFunctions.lit(true)\n val vanilla = untyped.as(\"df1\").join(untyped.as(\"df2\"), trivial).count()\n\n val typed = ds.joinInner(ds)(ds.colLeft('a) === ds.colLeft('a)).count().run\n vanilla ?= typed\n } }\n\n check(prop[Int, Int] _)\n }\n\n test(\"self join with unambiguous expression\") {\n def prop[\n A : TypedEncoder : CatalystNumeric : Ordering,\n B : TypedEncoder : Ordering\n ](data: List[X3[A, A, B]]): Prop = allowAmbiguousJoin {\n val ds = TypedDataset.create(data)\n\n val df1 = ds.dataset.alias(\"df1\")\n val df2 = ds.dataset.alias(\"df2\")\n\n val vanilla = df1.join(df2,\n (sparkFunctions.col(\"df1.a\") + sparkFunctions.col(\"df1.b\")) ===\n (sparkFunctions.col(\"df2.a\") + sparkFunctions.col(\"df2.b\"))).count()\n\n val typed = ds.joinInner(ds)(\n (ds.colLeft('a) + ds.colLeft('b)) === (ds.colRight('a) + ds.colRight('b))\n ).count().run()\n\n vanilla ?= typed\n }\n\n check(prop[Int, Int] _)\n }\n\n test(\"Do you want ambiguous self join? This is how you get ambiguous self join.\") {\n def prop[\n A : TypedEncoder : CatalystNumeric : Ordering,\n B : TypedEncoder : Ordering\n ](data: List[X3[A, A, B]]): Prop =\n allowTrivialJoin { allowAmbiguousJoin {\n val ds = TypedDataset.create(data)\n\n // The point I'm making here is that it \"behaves just like Spark\". I\n // don't know (or really care about how) how Spark disambiguates that\n // internally...\n val vanilla = ds.dataset.join(ds.dataset,\n (ds.dataset(\"a\") + ds.dataset(\"b\")) ===\n (ds.dataset(\"a\") + ds.dataset(\"b\"))).count()\n\n val typed = ds.joinInner(ds)(\n (ds.col('a) + ds.col('b)) === (ds.col('a) + ds.col('b))\n ).count().run()\n\n vanilla ?= typed\n } }\n\n check(prop[Int, Int] _)\n }\n\n test(\"colLeft and colRight are equivalent to col outside of joins\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n ex4: TypedEncoder[X4[A, B, C, D]]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val selectedCol = dataset.select(dataset.col [A]('a)).collect().run().toVector\n val selectedColLeft = dataset.select(dataset.colLeft [A]('a)).collect().run().toVector\n val selectedColRight = dataset.select(dataset.colRight[A]('a)).collect().run().toVector\n\n (selectedCol ?= selectedColLeft) && (selectedCol ?= selectedColRight)\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[X2[Int, Int], Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[UdtEncodedClass, Int, Int, Int] _))\n }\n\n test(\"colLeft and colRight are equivalent to col outside of joins - via files (codegen)\") {\n def prop[A, B, C, D](data: Vector[X4[A, B, C, D]])(\n implicit\n ea: TypedEncoder[A],\n ex4: TypedEncoder[X4[A, B, C, D]]\n ): Prop = {\n TypedDataset.create(data).write.mode(\"overwrite\").parquet(\"./target/testData\")\n val dataset = TypedDataset.createUnsafe[X4[A, B, C, D]](session.read.parquet(\"./target/testData\"))\n val selectedCol = dataset.select(dataset.col [A]('a)).collect().run().toVector\n val selectedColLeft = dataset.select(dataset.colLeft [A]('a)).collect().run().toVector\n val selectedColRight = dataset.select(dataset.colRight[A]('a)).collect().run().toVector\n\n (selectedCol ?= selectedColLeft) && (selectedCol ?= selectedColRight)\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[X2[Int, Int], Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, Int] _))\n check(forAll(prop[UdtEncodedClass, Int, Int, Int] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/TypedDatasetSuite.scala", "content": "package frameless\n\nimport com.globalmentor.apache.hadoop.fs.BareLocalFileSystem\nimport org.apache.hadoop.fs.local.StreamingFS\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.{SQLContext, SparkSession}\nimport org.scalactic.anyvals.PosZInt\nimport org.scalatest.BeforeAndAfterAll\nimport org.scalatestplus.scalacheck.Checkers\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nimport scala.util.{Properties, Try}\nimport org.scalatest.funsuite.AnyFunSuite\n\ntrait SparkTesting { self: BeforeAndAfterAll =>\n\n val appID: String = new java.util.Date().toString + math.floor(math.random * 10E4).toLong.toString\n\n /**\n * Allows bare naked to be used instead of winutils for testing / dev\n */\n def registerFS(sparkConf: SparkConf): SparkConf = {\n if (System.getProperty(\"os.name\").startsWith(\"Windows\"))\n sparkConf.set(\"spark.hadoop.fs.file.impl\", classOf[BareLocalFileSystem].getName).\n set(\"spark.hadoop.fs.AbstractFileSystem.file.impl\", classOf[StreamingFS].getName)\n else\n sparkConf\n }\n\n val conf: SparkConf = registerFS(new SparkConf())\n .setMaster(\"local[*]\")\n .setAppName(\"test\")\n .set(\"spark.ui.enabled\", \"false\")\n .set(\"spark.app.id\", appID)\n\n private var s: SparkSession = _\n\n implicit def session: SparkSession = s\n implicit def sc: SparkContext = session.sparkContext\n implicit def sqlContext: SQLContext = session.sqlContext\n\n def registerOptimizations(sqlContext: SQLContext): Unit = { }\n\n def addSparkConfigProperties(config: SparkConf): Unit = { }\n\n override def beforeAll(): Unit = {\n assert(s == null)\n addSparkConfigProperties(conf)\n s = SparkSession.builder().config(conf).getOrCreate()\n registerOptimizations(sqlContext)\n }\n\n override def afterAll(): Unit = {\n if (s != null) {\n s.stop()\n s = null\n }\n }\n}\n\n\nclass TypedDatasetSuite extends AnyFunSuite with Checkers with BeforeAndAfterAll with SparkTesting {\n // Limit size of generated collections and number of checks to avoid OutOfMemoryError\n implicit override val generatorDrivenConfig: PropertyCheckConfiguration = {\n def getPosZInt(name: String, default: PosZInt) = Properties.envOrNone(s\"FRAMELESS_GEN_${name}\")\n .flatMap(s => Try(s.toInt).toOption)\n .flatMap(PosZInt.from)\n .getOrElse(default)\n PropertyCheckConfiguration(\n sizeRange = getPosZInt(\"SIZE_RANGE\", PosZInt(20)),\n minSize = getPosZInt(\"MIN_SIZE\", PosZInt(0))\n )\n }\n\n implicit val sparkDelay: SparkDelay[Job] = Job.framelessSparkDelayForJob\n\n def approximatelyEqual[A](a: A, b: A)(implicit numeric: Numeric[A]): Prop = {\n val da = numeric.toDouble(a)\n val db = numeric.toDouble(b)\n val epsilon = 1E-6\n // Spark has a weird behaviour concerning expressions that should return Inf\n // Most of the time they return NaN instead, for instance stddev of Seq(-7.827553978923477E227, -5.009124275715786E153)\n if((da.isNaN || da.isInfinity) && (db.isNaN || db.isInfinity)) proved\n else if (\n (da - db).abs < epsilon ||\n (da - db).abs < da.abs / 100)\n proved\n else falsified :| s\"Expected $a but got $b, which is more than 1% off and greater than epsilon = $epsilon.\"\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/UdtEncodedClass.scala", "content": "package frameless\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArrayData}\nimport org.apache.spark.sql.types._\nimport org.apache.spark.sql.FramelessInternals.UserDefinedType\n\n@SQLUserDefinedType(udt = classOf[UdtEncodedClassUdt])\nclass UdtEncodedClass(val a: Int, val b: Array[Double]) {\n override def equals(other: Any): Boolean = other match {\n case that: UdtEncodedClass => a == that.a && java.util.Arrays.equals(b, that.b)\n case _ => false\n }\n\n override def hashCode(): Int = {\n val state = Seq[Any](a, b)\n state.map(_.hashCode()).foldLeft(0)((a, b) => 31 * a + b)\n }\n\n override def toString = s\"UdtEncodedClass($a, $b)\"\n}\n\nobject UdtEncodedClass {\n implicit val udtForUdtEncodedClass = new UdtEncodedClassUdt\n}\n\nclass UdtEncodedClassUdt extends UserDefinedType[UdtEncodedClass] {\n def sqlType: DataType = {\n StructType(Seq(\n StructField(\"a\", IntegerType, nullable = false),\n StructField(\"b\", ArrayType(DoubleType, containsNull = false), nullable = false)\n ))\n }\n\n def serialize(obj: UdtEncodedClass): InternalRow = {\n val row = new GenericInternalRow(3)\n row.setInt(0, obj.a)\n row.update(1, UnsafeArrayData.fromPrimitiveArray(obj.b))\n row\n }\n\n def deserialize(datum: Any): UdtEncodedClass = datum match {\n case row: InternalRow => new UdtEncodedClass(row.getInt(0), row.getArray(1).toDoubleArray())\n }\n\n def userClass: Class[UdtEncodedClass] = classOf[UdtEncodedClass]\n}\n" }, { "path": "dataset/src/test/scala/frameless/WithColumnTest.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\n\nclass WithColumnTest extends TypedDatasetSuite {\n import WithColumnTest._\n\n test(\"fail to compile on missing value\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XMissing] = f.withColumn[XMissing](f('j) === 10)\"\"\"\n }\n }\n\n test(\"fail to compile on different column name\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XDifferentColumnName] = f.withColumn[XDifferentColumnName](f('j) === 10)\"\"\"\n }\n }\n\n test(\"fail to compile on added column name\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XAdded] = f.withColumn[XAdded](f('j) === 10)\"\"\"\n }\n }\n\n test(\"fail to compile on wrong typed column\") {\n val f: TypedDataset[X] = TypedDataset.create(X(1,1) :: X(1,1) :: X(1,10) :: Nil)\n illTyped {\n \"\"\"val fNew: TypedDataset[XWrongType] = f.withColumn[XWrongType](f('j) === 10)\"\"\"\n }\n }\n\n test(\"append four columns\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d = TypedDataset.create(X1(value) :: Nil)\n val d1 = d.withColumn[X2[A, A]](d('a))\n val d2 = d1.withColumn[X3[A, A, A]](d1('b))\n val d3 = d2.withColumn[X4[A, A, A, A]](d2('c))\n val d4 = d3.withColumn[X5[A, A, A, A, A]](d3('d))\n\n X5(value, value, value, value, value) ?= d4.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n\n test(\"update in place\") {\n def prop[A : TypedEncoder](startValue: A, replaceValue: A): Prop = {\n val d = TypedDataset.create(X2(startValue, replaceValue) :: Nil)\n\n val X2(a, b) = d.withColumnReplaced('a, d('b))\n .collect()\n .run()\n .head\n\n a ?= b\n }\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n}\n\nobject WithColumnTest {\n case class X(i: Int, j: Int)\n case class XMissing(i: Int, k: Boolean)\n case class XDifferentColumnName(i: Int, ji: Int, k: Boolean)\n case class XAdded(i: Int, j: Int, k: Boolean, l: Int)\n case class XWrongType(i: Int, j: Int, k: Int)\n case class XGood(i: Int, j: Int, k: Boolean)\n}\n" }, { "path": "dataset/src/test/scala/frameless/WithColumnTupledTest.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass WithColumnTupledTest extends TypedDatasetSuite {\n test(\"append five columns\") {\n def prop[A: TypedEncoder](value: A): Prop = {\n val d = TypedDataset.create(X1(value) :: Nil)\n val d1 = d.withColumnTupled(d('a))\n val d2 = d1.withColumnTupled(d1('_1))\n val d3 = d2.withColumnTupled(d2('_2))\n val d4 = d3.withColumnTupled(d3('_3))\n val d5 = d4.withColumnTupled(d4('_4))\n\n (value, value, value, value, value, value) ?= d5.collect().run().head\n }\n\n check(prop[Int] _)\n check(prop[Long] _)\n check(prop[String] _)\n check(prop[SQLDate] _)\n check(prop[Option[X1[Boolean]]] _)\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/XN.scala", "content": "package frameless\n\nimport org.scalacheck.{Arbitrary, Cogen}\n\ncase class X1[A](a: A)\n\nobject X1 {\n implicit def arbitrary[A: Arbitrary]: Arbitrary[X1[A]] =\n Arbitrary(implicitly[Arbitrary[A]].arbitrary.map(X1(_)))\n\n implicit def cogen[A](implicit A: Cogen[A]): Cogen[X1[A]] =\n A.contramap(_.a)\n\n implicit def ordering[A: Ordering]: Ordering[X1[A]] = Ordering[A].on(_.a)\n}\n\ncase class X2[A, B](a: A, b: B)\n\nobject X2 {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary]: Arbitrary[X2[A, B]] =\n Arbitrary(Arbitrary.arbTuple2[A, B].arbitrary.map((X2.apply[A, B] _).tupled))\n\n implicit def cogen[A, B](implicit A: Cogen[A], B: Cogen[B]): Cogen[X2[A, B]] =\n Cogen.tuple2(A, B).contramap(x => (x.a, x.b))\n\n implicit def ordering[A: Ordering, B: Ordering]: Ordering[X2[A, B]] = Ordering.Tuple2[A, B].on(x => (x.a, x.b))\n}\n\ncase class X3[A, B, C](a: A, b: B, c: C)\n\nobject X3 {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary]: Arbitrary[X3[A, B, C]] =\n Arbitrary(Arbitrary.arbTuple3[A, B, C].arbitrary.map((X3.apply[A, B, C] _).tupled))\n\n implicit def cogen[A, B, C](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C]): Cogen[X3[A, B, C]] =\n Cogen.tuple3(A, B, C).contramap(x => (x.a, x.b, x.c))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering]: Ordering[X3[A, B, C]] =\n Ordering.Tuple3[A, B, C].on(x => (x.a, x.b, x.c))\n}\n\ncase class X3U[A, B, C](a: A, b: B, u: Unit, c: C)\n\nobject X3U {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary]: Arbitrary[X3U[A, B, C]] =\n Arbitrary(Arbitrary.arbTuple3[A, B, C].arbitrary.map(x => X3U[A, B, C](x._1, x._2, (), x._3)))\n\n implicit def cogen[A, B, C](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C]): Cogen[X3U[A, B, C]] =\n Cogen.tuple3(A, B, C).contramap(x => (x.a, x.b, x.c))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering]: Ordering[X3U[A, B, C]] =\n Ordering.Tuple3[A, B, C].on(x => (x.a, x.b, x.c))\n}\n\ncase class X3KV[A, B, C](key: A, value: B, c: C)\n\nobject X3KV {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary]: Arbitrary[X3KV[A, B, C]] =\n Arbitrary(Arbitrary.arbTuple3[A, B, C].arbitrary.map((X3KV.apply[A, B, C] _).tupled))\n\n implicit def cogen[A, B, C](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C]): Cogen[X3KV[A, B, C]] =\n Cogen.tuple3(A, B, C).contramap(x => (x.key, x.value, x.c))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering]: Ordering[X3KV[A, B, C]] =\n Ordering.Tuple3[A, B, C].on(x => (x.key, x.value, x.c))\n}\n\ncase class X4[A, B, C, D](a: A, b: B, c: C, d: D)\n\nobject X4 {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary, D: Arbitrary]: Arbitrary[X4[A, B, C, D]] =\n Arbitrary(Arbitrary.arbTuple4[A, B, C, D].arbitrary.map((X4.apply[A, B, C, D] _).tupled))\n\n implicit def cogen[A, B, C, D](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C], D: Cogen[D]): Cogen[X4[A, B, C, D]] =\n Cogen.tuple4(A, B, C, D).contramap(x => (x.a, x.b, x.c, x.d))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering, D: Ordering]: Ordering[X4[A, B, C, D]] =\n Ordering.Tuple4[A, B, C, D].on(x => (x.a, x.b, x.c, x.d))\n}\n\ncase class X5[A, B, C, D, E](a: A, b: B, c: C, d: D, e: E)\n\nobject X5 {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary, D: Arbitrary, E: Arbitrary]: Arbitrary[X5[A, B, C, D, E]] =\n Arbitrary(Arbitrary.arbTuple5[A, B, C, D, E].arbitrary.map((X5.apply[A, B, C, D, E] _).tupled))\n\n implicit def cogen[A, B, C, D, E](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C], D: Cogen[D], E: Cogen[E]): Cogen[X5[A, B, C, D, E]] =\n Cogen.tuple5(A, B, C, D, E).contramap(x => (x.a, x.b, x.c, x.d, x.e))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering, D: Ordering, E: Ordering]: Ordering[X5[A, B, C, D, E]] =\n Ordering.Tuple5[A, B, C, D, E].on(x => (x.a, x.b, x.c, x.d, x.e))\n}\n\ncase class X6[A, B, C, D, E, F](a: A, b: B, c: C, d: D, e: E, f: F)\n\nobject X6 {\n implicit def arbitrary[A: Arbitrary, B: Arbitrary, C: Arbitrary, D: Arbitrary, E: Arbitrary, F: Arbitrary]: Arbitrary[X6[A, B, C, D, E, F]] =\n Arbitrary(Arbitrary.arbTuple6[A, B, C, D, E, F].arbitrary.map((X6.apply[A, B, C, D, E, F] _).tupled))\n\n implicit def cogen[A, B, C, D, E, F](implicit A: Cogen[A], B: Cogen[B], C: Cogen[C], D: Cogen[D], E: Cogen[E], F: Cogen[F]): Cogen[X6[A, B, C, D, E, F]] =\n Cogen.tuple6(A, B, C, D, E, F).contramap(x => (x.a, x.b, x.c, x.d, x.e, x.f))\n\n implicit def ordering[A: Ordering, B: Ordering, C: Ordering, D: Ordering, E: Ordering, F: Ordering]: Ordering[X6[A, B, C, D, E, F]] =\n Ordering.Tuple6[A, B, C, D, E, F].on(x => (x.a, x.b, x.c, x.d, x.e, x.f))\n}" }, { "path": "dataset/src/test/scala/frameless/forward/CheckpointTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.{forAll, _}\n\n\nclass CheckpointTests extends TypedDatasetSuite {\n test(\"checkpoint\") {\n def prop[A: TypedEncoder](data: Vector[A], isEager: Boolean): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.sparkSession.sparkContext.setCheckpointDir(TEST_OUTPUT_DIR)\n\n dataset.checkpoint(isEager).run().queryExecution.toString() =?\n dataset.dataset.checkpoint(isEager).queryExecution.toString()\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/ColumnsTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.forAll\n\nclass ColumnsTests extends TypedDatasetSuite {\n test(\"columns\") {\n def prop(i: Int, s: String, b: Boolean, l: Long, d: Double, by: Byte): Prop = {\n val x1 = X1(i) :: Nil\n val x2 = X2(i, s) :: Nil\n val x3 = X3(i, s, b) :: Nil\n val x4 = X4(i, s, b, l) :: Nil\n val x5 = X5(i, s, b, l, d) :: Nil\n val x6 = X6(i, s, b, l, d, by) :: Nil\n\n val datasets = Seq(TypedDataset.create(x1), TypedDataset.create(x2),\n TypedDataset.create(x3), TypedDataset.create(x4),\n TypedDataset.create(x5), TypedDataset.create(x6))\n\n Prop.all(datasets.flatMap { dataset =>\n val columns = dataset.dataset.columns\n dataset.columns.map(col =>\n Prop.propBoolean(columns contains col)\n )\n }: _*)\n }\n\n check(forAll(prop _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/CountTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass CountTests extends TypedDatasetSuite {\n test(\"count\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop =\n TypedDataset.create(data).count().run() ?= data.size.toLong\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/DistinctTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport math.Ordering\n\nclass DistinctTests extends TypedDatasetSuite {\n test(\"distinct\") {\n // Comparison done with `.sorted` because order is not preserved by Spark for this operation.\n def prop[A: TypedEncoder : Ordering](data: Vector[A]): Prop =\n TypedDataset.create(data).distinct.collect().run().toVector.sorted ?= data.distinct.sorted\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/ExceptTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass ExceptTests extends TypedDatasetSuite {\n test(\"except\") {\n def prop[A: TypedEncoder](data1: Set[A], data2: Set[A]): Prop = {\n val dataset1 = TypedDataset.create(data1.toSeq)\n val dataset2 = TypedDataset.create(data2.toSeq)\n val datasetSubtract = dataset1.except(dataset2).collect().run().toVector\n val dataSubtract = data1.diff(data2)\n\n Prop.all(\n datasetSubtract.size ?= dataSubtract.size,\n datasetSubtract.toSet ?= dataSubtract\n )\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/FirstTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\n\nclass FirstTests extends TypedDatasetSuite with Matchers {\n test(\"first\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop =\n TypedDataset.create(data).firstOption().run() =? data.headOption\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"first on empty dataset should return None\") {\n TypedDataset.create(Vector[Int]()).firstOption().run() shouldBe None\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/ForeachTests.scala", "content": "package frameless\npackage forward\n\nimport org.apache.spark.util.CollectionAccumulator\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nimport scala.collection.JavaConverters._\n\nclass ForeachTests extends TypedDatasetSuite {\n test(\"foreach\") {\n def prop[A: Ordering: TypedEncoder](data: Vector[A]): Prop = {\n val accu = new CollectionAccumulator[A]()\n sc.register(accu)\n\n TypedDataset.create(data).foreach(accu.add).run()\n\n accu.value.asScala.toVector.sorted ?= data.sorted\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"foreachPartition\") {\n def prop[A: Ordering: TypedEncoder](data: Vector[A]): Prop = {\n val accu = new CollectionAccumulator[A]()\n sc.register(accu)\n\n TypedDataset.create(data).foreachPartition(_.foreach(accu.add)).run()\n\n accu.value.asScala.toVector.sorted ?= data.sorted\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/HeadTests.scala", "content": "package frameless.forward\n\nimport frameless.{TypedDataset, TypedDatasetSuite, TypedEncoder, TypedExpressionEncoder, X1}\nimport org.apache.spark.sql.SparkSession\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nimport scala.reflect.ClassTag\nimport org.scalatest.matchers.should.Matchers\n\nclass HeadTests extends TypedDatasetSuite with Matchers {\n def propArray[A: TypedEncoder : ClassTag : Ordering](data: Vector[X1[A]])(implicit c: SparkSession): Prop = {\n import c.implicits._\n if(data.nonEmpty) {\n val tds = TypedDataset.\n create(c.createDataset(data)(\n TypedExpressionEncoder.apply[X1[A]]\n ).orderBy($\"a\".desc))\n (tds.headOption().run().get ?= data.max).\n &&(tds.head(1).run().head ?= data.max).\n &&(tds.head(4).run().toVector ?=\n data.sortBy(_.a)(implicitly[Ordering[A]].reverse).take(4))\n } else Prop.passed\n }\n\n test(\"headOption(), head(1), and head(4)\") {\n check(propArray[Int] _)\n check(propArray[Char] _)\n check(propArray[String] _)\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/InputFilesTests.scala", "content": "package frameless\n\nimport java.util.UUID\n\nimport org.apache.spark.sql.SparkSession\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\n\nclass InputFilesTests extends TypedDatasetSuite with Matchers {\n test(\"inputFiles\") {\n\n def propText[A: TypedEncoder](data: Vector[A]): Prop = {\n val filePath = s\"$TEST_OUTPUT_DIR/${UUID.randomUUID()}.txt\"\n\n TypedDataset.create(data).dataset.write.text(filePath)\n val dataset = TypedDataset.create(implicitly[SparkSession].sparkContext.textFile(filePath))\n\n dataset.inputFiles sameElements dataset.dataset.inputFiles\n }\n\n def propCsv[A: TypedEncoder](data: Vector[A]): Prop = {\n val filePath = s\"$TEST_OUTPUT_DIR/${UUID.randomUUID()}.csv\"\n val inputDataset = TypedDataset.create(data)\n inputDataset.dataset.write.csv(filePath)\n\n val dataset = TypedDataset.createUnsafe(\n implicitly[SparkSession].sqlContext.read.schema(inputDataset.schema).csv(filePath))\n\n dataset.inputFiles sameElements dataset.dataset.inputFiles\n }\n\n def propJson[A: TypedEncoder](data: Vector[A]): Prop = {\n val filePath = s\"$TEST_OUTPUT_DIR/${UUID.randomUUID()}.json\"\n val inputDataset = TypedDataset.create(data)\n inputDataset.dataset.write.json(filePath)\n\n val dataset = TypedDataset.createUnsafe(\n implicitly[SparkSession].sqlContext.read.schema(inputDataset.schema).json(filePath))\n\n dataset.inputFiles sameElements dataset.dataset.inputFiles\n }\n\n check(forAll(propText[String] _))\n check(forAll(propCsv[String] _))\n check(forAll(propJson[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/IntersectTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport math.Ordering\n\nclass IntersectTests extends TypedDatasetSuite {\n test(\"intersect\") {\n def prop[A: TypedEncoder : Ordering](data1: Vector[A], data2: Vector[A]): Prop = {\n val dataset1 = TypedDataset.create(data1)\n val dataset2 = TypedDataset.create(data2)\n val datasetIntersect = dataset1.intersect(dataset2).collect().run().toVector\n\n // Vector `intersect` is the multiset intersection, while Spark throws away duplicates.\n val dataIntersect = data1.intersect(data2).distinct\n\n // Comparison done with `.sorted` because order is not preserved by Spark for this operation.\n datasetIntersect.sorted ?= dataIntersect.distinct.sorted\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/IsLocalTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass IsLocalTests extends TypedDatasetSuite {\n test(\"isLocal\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.isLocal ?= dataset.dataset.isLocal\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/IsStreamingTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass IsStreamingTests extends TypedDatasetSuite {\n test(\"isStreaming\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.isStreaming ?= dataset.dataset.isStreaming\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/LimitTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass LimitTests extends TypedDatasetSuite {\n test(\"limit\") {\n def prop[A: TypedEncoder](data: Vector[A], n: Int): Prop = (n >= 0) ==> {\n val dataset = TypedDataset.create(data).limit(n).collect().run()\n\n Prop.all(\n dataset.length ?= Math.min(data.length, n),\n dataset.forall(data.contains)\n )\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/QueryExecutionTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.{forAll, _}\n\nclass QueryExecutionTests extends TypedDatasetSuite {\n test(\"queryExecution\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create[A](data)\n\n dataset.queryExecution =? dataset.dataset.queryExecution\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/RandomSplitTests.scala", "content": "package frameless\n\nimport org.scalacheck.Arbitrary.arbitrary\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport scala.collection.JavaConverters._\nimport org.scalatest.matchers.should.Matchers\n\nclass RandomSplitTests extends TypedDatasetSuite with Matchers {\n\n val nonEmptyPositiveArray: Gen[Array[Double]] = Gen.nonEmptyListOf(Gen.posNum[Double]).map(_.toArray)\n\n test(\"randomSplit(weight, seed)\") {\n def prop[A: TypedEncoder : Arbitrary] = forAll(vectorGen[A], nonEmptyPositiveArray, arbitrary[Long]) {\n (data: Vector[A], weights: Array[Double], seed: Long) =>\n val dataset = TypedDataset.create(data)\n\n dataset.randomSplit(weights, seed).map(_.count().run()) sameElements\n dataset.dataset.randomSplit(weights, seed).map(_.count())\n }\n\n check(prop[Int])\n check(prop[String])\n }\n\n test(\"randomSplitAsList(weight, seed)\") {\n def prop[A: TypedEncoder : Arbitrary] = forAll(vectorGen[A], nonEmptyPositiveArray, arbitrary[Long]) {\n (data: Vector[A], weights: Array[Double], seed: Long) =>\n val dataset = TypedDataset.create(data)\n\n dataset.randomSplitAsList(weights, seed).asScala.map(_.count().run()) sameElements\n dataset.dataset.randomSplitAsList(weights, seed).asScala.map(_.count())\n }\n\n check(prop[Int])\n check(prop[String])\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/SQLContextTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.{forAll, _}\n\nclass SQLContextTests extends TypedDatasetSuite {\n test(\"sqlContext\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create[A](data)\n\n dataset.sqlContext =? dataset.dataset.sqlContext\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/SparkSessionTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass SparkSessionTests extends TypedDatasetSuite {\n test(\"sparkSession\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create[A](data)\n\n dataset.sparkSession =? dataset.dataset.sparkSession\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/StorageLevelTests.scala", "content": "package frameless\n\nimport org.apache.spark.storage.StorageLevel\nimport org.apache.spark.storage.StorageLevel._\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Arbitrary, Gen}\n\nclass StorageLevelTests extends TypedDatasetSuite {\n\n val storageLevelGen: Gen[StorageLevel] = Gen.oneOf(Seq(NONE, DISK_ONLY, DISK_ONLY_2, MEMORY_ONLY,\n MEMORY_ONLY_2, MEMORY_ONLY_SER, MEMORY_ONLY_SER_2, MEMORY_AND_DISK,\n MEMORY_AND_DISK_2, MEMORY_AND_DISK_SER, MEMORY_AND_DISK_SER_2, OFF_HEAP))\n\n test(\"storageLevel\") {\n def prop[A: TypedEncoder : Arbitrary] = forAll(vectorGen[A], storageLevelGen) {\n (data: Vector[A], storageLevel: StorageLevel) =>\n val dataset = TypedDataset.create(data)\n if (storageLevel != StorageLevel.NONE)\n dataset.persist(storageLevel)\n\n dataset.count().run()\n\n dataset.storageLevel() ?= dataset.dataset.storageLevel\n }\n\n check(prop[Int])\n check(prop[String])\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/TakeTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport scala.reflect.ClassTag\n\nclass TakeTests extends TypedDatasetSuite {\n test(\"take\") {\n def prop[A: TypedEncoder](n: Int, data: Vector[A]): Prop =\n (n >= 0) ==> (TypedDataset.create(data).take(n).run().toVector =? data.take(n))\n\n def propArray[A: TypedEncoder: ClassTag](n: Int, data: Vector[X1[Array[A]]]): Prop =\n (n >= 0) ==> {\n Prop {\n TypedDataset.create(data).take(n).run().toVector.zip(data.take(n)).forall {\n case (X1(l), X1(r)) => l sameElements r\n }\n }\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n check(forAll(propArray[Int] _))\n check(forAll(propArray[String] _))\n check(forAll(propArray[Byte] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/ToJSONTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass ToJSONTests extends TypedDatasetSuite {\n test(\"toJSON\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.toJSON.collect().run() ?= dataset.dataset.toJSON.collect()\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/forward/ToLocalIteratorTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport scala.collection.JavaConverters._\nimport org.scalatest.matchers.should.Matchers\n\nclass ToLocalIteratorTests extends TypedDatasetSuite with Matchers {\n test(\"toLocalIterator\") {\n def prop[A: TypedEncoder](data: Vector[A]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.toLocalIterator().run().asScala.toIterator sameElements dataset.dataset.toLocalIterator().asScala.toIterator\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/UnionTests.scala", "content": "package frameless\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\n\nclass UnionTests extends TypedDatasetSuite {\n\n test(\"fail to compile on not aligned schema\") {\n val dataset1 = TypedDataset.create(Foo(1, 1) :: Nil)\n val dataset2 = TypedDataset.create(Wrong(1, 1, 1) :: Nil)\n\n illTyped {\n \"\"\"val fNew = dataset1 union dataset2 \"\"\"\n }\n }\n\n test(\"Union for simple data types\") {\n def prop[A: TypedEncoder](data1: Vector[A], data2: Vector[A]): Prop = {\n val dataset1 = TypedDataset.create(data1)\n val dataset2 = TypedDataset.create(data2)\n val datasetUnion = dataset1.union(dataset2).collect().run().toVector\n val dataUnion = data1.union(data2)\n\n datasetUnion ?= dataUnion\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"Align fields for case classes\") {\n def prop[A: TypedEncoder, B: TypedEncoder](data1: Vector[(A, B)], data2: Vector[(A, B)]): Prop = {\n\n val dataset1 = TypedDataset.create(data1.map((Foo.apply[A, B] _).tupled))\n val dataset2 = TypedDataset.create(data2.map { case (a, b) => Bar[A, B](b, a) })\n val datasetUnion = dataset1.union(dataset2).collect().run().map(foo => (foo.x, foo.y)).toVector\n val dataUnion = data1 union data2\n\n datasetUnion ?= dataUnion\n }\n\n check(forAll(prop[Int, String] _))\n check(forAll(prop[String, X1[Option[Long]]] _))\n }\n\n test(\"Align fields for different number of columns\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder](data1: Vector[(A, B, C)], data2: Vector[(A, B)]): Prop = {\n\n val dataset1 = TypedDataset.create(data2.map((Foo.apply[A, B] _).tupled))\n val dataset2 = TypedDataset.create(data1.map { case (a, b, c) => Baz[A, B, C](c, b, a) })\n val datasetUnion: Seq[(A, B)] = dataset1.union(dataset2).collect().run().map(foo => (foo.x, foo.y)).toVector\n val dataUnion = data2 union data1.map { case (a, b, _) => (a, b) }\n\n datasetUnion ?= dataUnion\n }\n\n check(forAll(prop[Option[Int], String, Array[Long]] _))\n check(forAll(prop[String, X1[Option[Int]], X2[String, Array[Int]]] _))\n }\n}\n\nfinal case class Foo[A, B](x: A, y: B)\nfinal case class Bar[A, B](y: B, x: A)\nfinal case class Baz[A, B, C](z: C, y: B, x: A)\nfinal case class Wrong[A, B, C](a: A, b: B, c: C)" }, { "path": "dataset/src/test/scala/frameless/forward/WriteStreamTests.scala", "content": "package frameless\n\nimport java.util.UUID\n\nimport org.apache.spark.sql.Encoder\nimport org.apache.spark.sql.execution.streaming.MemoryStream\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Arbitrary, Gen, Prop}\n\nclass WriteStreamTests extends TypedDatasetSuite {\n\n val genNested = for {\n d <- Arbitrary.arbitrary[Double]\n as <- Arbitrary.arbitrary[String]\n } yield Nested(d, as)\n\n val genOptionFieldsOnly = for {\n o1 <- Gen.option(Arbitrary.arbitrary[Int])\n o2 <- Gen.option(genNested)\n } yield OptionFieldsOnly(o1, o2)\n\n val genWriteExample = for {\n i <- Arbitrary.arbitrary[Int]\n s <- Arbitrary.arbitrary[String]\n on <- Gen.option(genNested)\n ooo <- Gen.option(genOptionFieldsOnly)\n } yield WriteExample(i, s, on, ooo)\n\n test(\"write csv\") {\n val spark = session\n import spark.implicits._\n def prop[A: TypedEncoder: Encoder](data: List[A]): Prop = {\n val uid = UUID.randomUUID()\n val uidNoHyphens = uid.toString.replace(\"-\", \"\")\n val filePath = s\"$TEST_OUTPUT_DIR/$uid}\"\n val checkpointPath = s\"$TEST_OUTPUT_DIR/checkpoint/$uid\"\n val inputStream = MemoryStream[A]\n val input = TypedDataset.create(inputStream.toDS())\n val inputter = input.writeStream.format(\"csv\").option(\"checkpointLocation\", s\"$checkpointPath/input\").start(filePath)\n inputStream.addData(data)\n inputter.processAllAvailable()\n val dataset = TypedDataset.createUnsafe(sqlContext.readStream.schema(input.schema).csv(filePath))\n\n val tester = dataset\n .writeStream\n .option(\"checkpointLocation\", s\"$checkpointPath/tester\")\n .format(\"memory\")\n .queryName(s\"testCsv_$uidNoHyphens\")\n .start()\n tester.processAllAvailable()\n val output = spark.table(s\"testCsv_$uidNoHyphens\").as[A]\n TypedDataset.create(data).collect().run().groupBy(identity) ?= output.collect().groupBy(identity).map { case (k, arr) => (k, arr.toSeq) }\n }\n\n check(forAll(Gen.nonEmptyListOf(Gen.alphaNumStr.suchThat(_.nonEmpty)))(prop[String]))\n check(forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int]))(prop[Int]))\n }\n\n test(\"write parquet\") {\n val spark = session\n import spark.implicits._\n def prop[A: TypedEncoder: Encoder](data: List[A]): Prop = {\n val uid = UUID.randomUUID()\n val uidNoHyphens = uid.toString.replace(\"-\", \"\")\n val filePath = s\"$TEST_OUTPUT_DIR/$uid}\"\n val checkpointPath = s\"$TEST_OUTPUT_DIR/checkpoint/$uid\"\n val inputStream = MemoryStream[A]\n val input = TypedDataset.create(inputStream.toDS())\n val inputter = input.writeStream.format(\"parquet\").option(\"checkpointLocation\", s\"$checkpointPath/input\").start(filePath)\n inputStream.addData(data)\n inputter.processAllAvailable()\n val dataset = TypedDataset.createUnsafe(sqlContext.readStream.schema(input.schema).parquet(filePath))\n\n val tester = dataset\n .writeStream\n .option(\"checkpointLocation\", s\"$checkpointPath/tester\")\n .format(\"memory\")\n .queryName(s\"testParquet_$uidNoHyphens\")\n .start()\n tester.processAllAvailable()\n val output = spark.table(s\"testParquet_$uidNoHyphens\").as[A]\n TypedDataset.create(data).collect().run().groupBy(identity) ?= output.collect().groupBy(identity).map { case (k, arr) => (k, arr.toSeq) }\n }\n\n check(forAll(Gen.nonEmptyListOf(genWriteExample))(prop[WriteExample]))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/forward/WriteTests.scala", "content": "package frameless\n\nimport java.util.UUID\n\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Arbitrary, Gen, Prop}\n\nclass WriteTests extends TypedDatasetSuite {\n\n val genNested = for {\n d <- Arbitrary.arbitrary[Double]\n as <- Arbitrary.arbitrary[String]\n } yield Nested(d, as)\n\n val genOptionFieldsOnly = for {\n o1 <- Gen.option(Arbitrary.arbitrary[Int])\n o2 <- Gen.option(genNested)\n } yield OptionFieldsOnly(o1, o2)\n\n val genWriteExample = for {\n i <- Arbitrary.arbitrary[Int]\n s <- Arbitrary.arbitrary[String]\n on <- Gen.option(genNested)\n ooo <- Gen.option(genOptionFieldsOnly)\n } yield WriteExample(i, s, on, ooo)\n\n test(\"write csv\") {\n def prop[A: TypedEncoder](data: List[A]): Prop = {\n val filePath = s\"$TEST_OUTPUT_DIR/${UUID.randomUUID()}\"\n val input = TypedDataset.create(data)\n input.write.csv(filePath)\n\n val dataset = TypedDataset.createUnsafe(sqlContext.read.schema(input.schema).csv(filePath))\n\n dataset.collect().run().groupBy(identity) ?= input.collect().run().groupBy(identity)\n }\n\n check(forAll(Gen.listOf(Gen.alphaNumStr.suchThat(_.nonEmpty)))(prop[String]))\n check(forAll(prop[Int] _))\n }\n\n test(\"write parquet\") {\n def prop[A: TypedEncoder](data: List[A]): Prop = {\n val filePath = s\"$TEST_OUTPUT_DIR/${UUID.randomUUID()}\"\n val input = TypedDataset.create(data)\n input.write.parquet(filePath)\n\n val dataset = TypedDataset.createUnsafe(sqlContext.read.schema(input.schema).parquet(filePath))\n\n dataset.collect().run().groupBy(identity) ?= input.collect().run().groupBy(identity)\n }\n\n check(forAll(Gen.listOf(genWriteExample))(prop[WriteExample]))\n }\n}\n\ncase class Nested(i: Double, v: String)\ncase class OptionFieldsOnly(o1: Option[Int], o2: Option[Nested])\ncase class WriteExample(i: Int, s: String, on: Option[Nested], ooo: Option[OptionFieldsOnly])\n" }, { "path": "dataset/src/test/scala/frameless/functions/AggregateFunctionsTests.scala", "content": "package frameless\npackage functions\n\nimport frameless.{TypedAggregate, TypedColumn}\nimport frameless.functions.aggregate._\nimport org.apache.spark.sql.{Column, Encoder}\nimport org.scalacheck.{Gen, Prop}\nimport org.scalacheck.Prop._\nimport org.scalatest.exceptions.GeneratorDrivenPropertyCheckFailedException\n\nclass AggregateFunctionsTests extends TypedDatasetSuite {\n def sparkSchema[A: TypedEncoder, U](f: TypedColumn[X1[A], A] => TypedAggregate[X1[A], U]): Prop = {\n val df = TypedDataset.create[X1[A]](Nil)\n val col = f(df.col('a))\n\n val sumDf = df.agg(col)\n\n TypedExpressionEncoder.targetStructType(sumDf.encoder) ?= sumDf.dataset.schema\n }\n\n test(\"sum\") {\n case class Sum4Tests[A, B](sum: Seq[A] => B)\n\n def prop[A: TypedEncoder, Out: TypedEncoder : Numeric](xs: List[A])(\n implicit\n summable: CatalystSummable[A, Out],\n summer: Sum4Tests[A, Out]\n ): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetSum: List[Out] = dataset.agg(sum(A)).collect().run().toList\n\n datasetSum match {\n case x :: Nil => approximatelyEqual(summer.sum(xs), x)\n case other => falsified\n }\n }\n\n // Replicate Spark's behaviour : Ints and Shorts are cast to Long\n // https://github.com/apache/spark/blob/7eb2ca8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L37\n implicit def summerDecimal = Sum4Tests[BigDecimal, BigDecimal](_.sum)\n implicit def summerDouble = Sum4Tests[Double, Double](_.sum)\n implicit def summerLong = Sum4Tests[Long, Long](_.sum)\n implicit def summerInt = Sum4Tests[Int, Long](_.map(_.toLong).sum)\n implicit def summerShort = Sum4Tests[Short, Long](_.map(_.toLong).sum)\n\n check(forAll(prop[BigDecimal, BigDecimal] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[Double, Double] _))\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[Short, Long] _))\n\n check(sparkSchema[BigDecimal, BigDecimal](sum))\n check(sparkSchema[Long, Long](sum))\n check(sparkSchema[Int, Long](sum))\n check(sparkSchema[Double, Double](sum))\n check(sparkSchema[Short, Long](sum))\n }\n\n test(\"sumDistinct\") {\n case class Sum4Tests[A, B](sum: Seq[A] => B)\n\n def prop[A: TypedEncoder, Out: TypedEncoder : Numeric](xs: List[A])(\n implicit\n summable: CatalystSummable[A, Out],\n summer: Sum4Tests[A, Out]\n ): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetSum: List[Out] = dataset.agg(sumDistinct(A)).collect().run().toList\n\n datasetSum match {\n case x :: Nil => approximatelyEqual(summer.sum(xs), x)\n case other => falsified\n }\n }\n\n // Replicate Spark's behaviour : Ints and Shorts are cast to Long\n // https://github.com/apache/spark/blob/7eb2ca8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L37\n implicit def summerLong = Sum4Tests[Long, Long](_.toSet.sum)\n implicit def summerInt = Sum4Tests[Int, Long]( x => x.toSet.map((_:Int).toLong).sum)\n implicit def summerShort = Sum4Tests[Short, Long](x => x.toSet.map((_:Short).toLong).sum)\n\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[Short, Long] _))\n\n check(sparkSchema[Long, Long](sum))\n check(sparkSchema[Int, Long](sum))\n check(sparkSchema[Short, Long](sum))\n }\n\n test(\"avg\") {\n case class Averager4Tests[A, B](avg: Seq[A] => B)\n\n def prop[A: TypedEncoder, Out: TypedEncoder : Numeric](xs: List[A])(\n implicit\n averageable: CatalystAverageable[A, Out],\n averager: Averager4Tests[A, Out]\n ): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetAvg: Vector[Out] = dataset.agg(avg(A)).collect().run().toVector\n\n if (datasetAvg.size > 2) falsified\n else xs match {\n case Nil => datasetAvg ?= Vector()\n case _ :: _ => datasetAvg.headOption match {\n case Some(x) => approximatelyEqual(averager.avg(xs), x)\n case None => falsified\n }\n }\n }\n\n // Replicate Spark's behaviour : If the datatype isn't BigDecimal cast type to Double\n // https://github.com/apache/spark/blob/7eb2ca8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala#L50\n implicit def averageDecimal = Averager4Tests[BigDecimal, BigDecimal](as => as.sum/as.size)\n implicit def averageDouble = Averager4Tests[Double, Double](as => as.sum/as.size)\n implicit def averageLong = Averager4Tests[Long, Double](as => as.map(_.toDouble).sum/as.size)\n implicit def averageInt = Averager4Tests[Int, Double](as => as.map(_.toDouble).sum/as.size)\n implicit def averageShort = Averager4Tests[Short, Double](as => as.map(_.toDouble).sum/as.size)\n\n /* under 3.4 an oddity was detected:\n Falsified after 2 successful property evaluations.\n Location: (AggregateFunctionsTests.scala:127)\n [info] Occurred when passed generated values (\n [info] arg0 = List(\"-1\", \"9223372036854775807\", \"-9223372036854775808\")\n [info] )\n which is odd given it's strings and not the Long's that should have been there, but also not seemingly reproducible with just longs\n */\n tolerantRun(_.isInstanceOf[GeneratorDrivenPropertyCheckFailedException]) {\n check(forAll(prop[BigDecimal, BigDecimal] _))\n check(forAll(prop[Double, Double] _))\n check(forAll(prop[Long, Double] _))\n check(forAll(prop[Int, Double] _))\n check(forAll(prop[Short, Double] _))\n }\n }\n\n test(\"stddev and variance\") {\n def prop[A: TypedEncoder : CatalystVariance : Numeric](xs: List[A]): Prop = {\n val numeric = implicitly[Numeric[A]]\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetStdOpt = dataset.agg(stddev(A)).collect().run().toVector.headOption\n val datasetVarOpt = dataset.agg(variance(A)).collect().run().toVector.headOption\n\n val std = sc.parallelize(xs.map(implicitly[Numeric[A]].toDouble)).sampleStdev()\n val `var` = sc.parallelize(xs.map(implicitly[Numeric[A]].toDouble)).sampleVariance()\n\n (datasetStdOpt, datasetVarOpt) match {\n case (Some(datasetStd), Some(datasetVar)) =>\n approximatelyEqual(datasetStd, std) && approximatelyEqual(datasetVar, `var`)\n case _ => proved\n }\n }\n\n check(forAll(prop[Short] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"litAggr\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder](xs: List[A], b: B, c: C): Prop = {\n val dataset = TypedDataset.create(xs)\n val (r1, rb, rc, rcount) = dataset.agg(count().lit(1), litAggr(b), litAggr(c), count()).collect().run().head\n (rcount ?= xs.size.toLong) && (r1 ?= 1) && (rb ?= b) && (rc ?= c)\n }\n\n check(forAll(prop[Boolean, Int, String] _))\n check(forAll(prop[Option[Boolean], Vector[Option[Vector[Char]]], Long] _))\n }\n\n test(\"count\") {\n def prop[A: TypedEncoder](xs: List[A]): Prop = {\n val dataset = TypedDataset.create(xs)\n val Vector(datasetCount) = dataset.agg(count()).collect().run().toVector\n\n datasetCount ?= xs.size.toLong\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Byte] _))\n }\n\n test(\"count('a)\") {\n def prop[A: TypedEncoder](xs: List[A]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n val datasetCount = dataset.agg(count(A)).collect().run()\n\n datasetCount ?= List(xs.size.toLong)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Byte] _))\n }\n\n test(\"max\") {\n def prop[A: TypedEncoder: CatalystOrdered](xs: List[A])(implicit o: Ordering[A]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n val datasetMax = dataset.agg(max(A)).collect().run().toList\n\n datasetMax ?= xs.reduceOption[A](o.max).toList\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n test(\"max with follow up multiplication\") {\n def prop(xs: List[Long]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[Long]('a)\n val datasetMax = dataset.agg(max(A) * 2).collect().run().headOption\n\n datasetMax ?= (if(xs.isEmpty) None else Some(xs.max * 2))\n }\n\n check(forAll(prop _))\n }\n\n test(\"min\") {\n def prop[A: TypedEncoder: CatalystOrdered](xs: List[A])(implicit o: Ordering[A]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetMin = dataset.agg(min(A)).collect().run().toList\n\n datasetMin ?= xs.reduceOption[A](o.min).toList\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n test(\"first\") {\n def prop[A: TypedEncoder](xs: List[A]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetFirst = dataset.agg(first(A)).collect().run().toList\n\n datasetFirst ?= xs.headOption.toList\n }\n\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n test(\"last\") {\n def prop[A: TypedEncoder](xs: List[A]): Prop = {\n val dataset = TypedDataset.create(xs.map(X1(_)))\n val A = dataset.col[A]('a)\n\n val datasetLast = dataset.agg(last(A)).collect().run().toList\n\n datasetLast ?= xs.lastOption.toList\n }\n\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n // Generator for simplified and focused aggregation examples\n def getLowCardinalityKVPairs: Gen[Vector[(Int, Int)]] = {\n val kvPairGen: Gen[(Int, Int)] = for {\n k <- Gen.const(1) // key\n v <- Gen.choose(10, 100) // values\n } yield (k, v)\n\n Gen.listOfN(200, kvPairGen).map(_.toVector)\n }\n\n test(\"countDistinct\") {\n check {\n forAll(getLowCardinalityKVPairs) { xs: Vector[(Int, Int)] =>\n val tds = TypedDataset.create(xs)\n val tdsRes: Seq[(Int, Long)] = tds.groupBy(tds('_1)).agg(countDistinct(tds('_2))).collect().run()\n tdsRes.toMap ?= xs.groupBy(_._1).mapValues(_.map(_._2).distinct.size.toLong).toSeq.toMap\n }\n }\n }\n\n test(\"approxCountDistinct\") {\n // Simple version of #approximatelyEqual()\n // Default maximum estimation error of HyperLogLog in Spark is 5%\n def approxEqual(actual: Long, estimated: Long, allowedDeviationPercentile: Double = 0.05): Boolean = {\n val delta: Long = Math.abs(actual - estimated)\n delta / actual.toDouble < allowedDeviationPercentile * 2\n }\n\n check {\n forAll(getLowCardinalityKVPairs) { xs: Vector[(Int, Int)] =>\n val tds = TypedDataset.create(xs)\n val tdsRes: Seq[(Int, Long, Long)] =\n tds.groupBy(tds('_1)).agg(countDistinct(tds('_2)), approxCountDistinct(tds('_2))).collect().run()\n tdsRes.forall { case (_, v1, v2) => approxEqual(v1, v2) }\n }\n }\n\n check {\n forAll(getLowCardinalityKVPairs) { xs: Vector[(Int, Int)] =>\n val tds = TypedDataset.create(xs)\n val allowedError = 0.1 // 10%\n val tdsRes: Seq[(Int, Long, Long)] =\n tds.groupBy(tds('_1)).agg(countDistinct(tds('_2)), approxCountDistinct(tds('_2), allowedError)).collect().run()\n tdsRes.forall { case (_, v1, v2) => approxEqual(v1, v2, allowedError) }\n }\n }\n }\n\n test(\"collectList\") {\n def prop[A: TypedEncoder : Ordering](xs: List[X2[A, A]]): Prop = {\n val tds = TypedDataset.create(xs)\n val tdsRes: Seq[(A, Vector[A])] = tds.groupBy(tds('a)).agg(collectList(tds('b))).collect().run()\n\n tdsRes.toMap.map { case (k, v) => k -> v.sorted } ?= xs.groupBy(_.a).map { case (k, v) => k -> v.map(_.b).toVector.sorted }\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n test(\"collectSet\") {\n def prop[A: TypedEncoder : Ordering](xs: List[X2[A, A]]): Prop = {\n val tds = TypedDataset.create(xs)\n val tdsRes: Seq[(A, Vector[A])] = tds.groupBy(tds('a)).agg(collectSet(tds('b))).collect().run()\n\n tdsRes.toMap.map { case (k, v) => k -> v.toSet } ?= xs.groupBy(_.a).map { case (k, v) => k -> v.map(_.b).toSet }\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[String] _))\n }\n\n test(\"lit\") {\n def prop[A: TypedEncoder](xs: List[X1[A]], l: A): Prop = {\n val tds = TypedDataset.create(xs)\n tds.select(tds('a), lit(l)).collect().run() ?= xs.map(x => (x.a, l))\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Vector[Vector[Int]]] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Vector[Byte]] _))\n check(forAll(prop[String] _))\n check(forAll(prop[Vector[Long]] _))\n check(forAll(prop[BigDecimal] _))\n }\n\n\n def bivariatePropTemplate[A: TypedEncoder, B: TypedEncoder]\n (\n xs: List[X3[Int, A, B]]\n )\n (\n framelessFun: (TypedColumn[X3[Int, A, B], A], TypedColumn[X3[Int, A, B], B]) => TypedAggregate[X3[Int, A, B], Option[Double]],\n sparkFun: (Column, Column) => Column\n )\n (\n implicit\n encEv: Encoder[(Int, A, B)],\n encEv2: Encoder[(Int,Option[Double])],\n evCanBeDoubleA: CatalystCast[A, Double],\n evCanBeDoubleB: CatalystCast[B, Double]\n ): Prop = {\n\n val tds = TypedDataset.create(xs)\n // Typed implementation of bivar stats function\n val tdBivar = tds.groupBy(tds('a)).agg(framelessFun(tds('b), tds('c))).deserialized.map(kv =>\n (kv._1, kv._2.flatMap(DoubleBehaviourUtils.nanNullHandler))\n ).collect().run()\n\n val cDF = session.createDataset(xs.map(x => (x.a, x.b, x.c)))\n // Comparison implementation of bivar stats functions\n val compBivar = cDF\n .groupBy(cDF(\"_1\"))\n .agg(sparkFun(cDF(\"_2\"), cDF(\"_3\")))\n .map(\n row => {\n val grp = row.getInt(0)\n (grp, DoubleBehaviourUtils.nanNullHandler(row.get(1)))\n }\n )\n\n // Should be the same\n tdBivar.toMap ?= compBivar.collect().toMap\n }\n\n def univariatePropTemplate[A: TypedEncoder]\n (\n xs: List[X2[Int, A]]\n )\n (\n framelessFun: (TypedColumn[X2[Int, A], A]) => TypedAggregate[X2[Int, A], Option[Double]],\n sparkFun: (Column) => Column\n )\n (\n implicit\n encEv: Encoder[(Int, A)],\n encEv2: Encoder[(Int,Option[Double])],\n evCanBeDoubleA: CatalystCast[A, Double]\n ): Prop = {\n\n val tds = TypedDataset.create(xs)\n //typed implementation of univariate stats function\n val tdUnivar = tds.groupBy(tds('a)).agg(framelessFun(tds('b))).deserialized.map(kv =>\n (kv._1, kv._2.flatMap(DoubleBehaviourUtils.nanNullHandler))\n ).collect().run()\n\n val cDF = session.createDataset(xs.map(x => (x.a, x.b)))\n // Comparison implementation of bivar stats functions\n val compUnivar = cDF\n .groupBy(cDF(\"_1\"))\n .agg(sparkFun(cDF(\"_2\")))\n .map(\n row => {\n val grp = row.getInt(0)\n (grp, DoubleBehaviourUtils.nanNullHandler(row.get(1)))\n }\n )\n\n // Should be the same\n tdUnivar.toMap ?= compUnivar.collect().toMap\n }\n\n test(\"corr\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder, B: TypedEncoder](xs: List[X3[Int, A, B]])(\n implicit\n encEv: Encoder[(Int, A, B)],\n evCanBeDoubleA: CatalystCast[A, Double],\n evCanBeDoubleB: CatalystCast[B, Double]\n ): Prop = bivariatePropTemplate(xs)(corr[A,B,X3[Int, A, B]],org.apache.spark.sql.functions.corr)\n\n check(forAll(prop[Double, Double] _))\n check(forAll(prop[Double, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[BigDecimal, Byte] _))\n }\n\n test(\"covar_pop\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder, B: TypedEncoder](xs: List[X3[Int, A, B]])(\n implicit\n encEv: Encoder[(Int, A, B)],\n evCanBeDoubleA: CatalystCast[A, Double],\n evCanBeDoubleB: CatalystCast[B, Double]\n ): Prop = bivariatePropTemplate(xs)(\n covarPop[A, B, X3[Int, A, B]],\n org.apache.spark.sql.functions.covar_pop\n )\n\n check(forAll(prop[Double, Double] _))\n check(forAll(prop[Double, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[BigDecimal, Byte] _))\n }\n\n test(\"covar_samp\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder, B: TypedEncoder](xs: List[X3[Int, A, B]])(\n implicit\n encEv: Encoder[(Int, A, B)],\n evCanBeDoubleA: CatalystCast[A, Double],\n evCanBeDoubleB: CatalystCast[B, Double]\n ): Prop = bivariatePropTemplate(xs)(\n covarSamp[A, B, X3[Int, A, B]],\n org.apache.spark.sql.functions.covar_samp\n )\n\n check(forAll(prop[Double, Double] _))\n check(forAll(prop[Double, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[BigDecimal, Byte] _))\n }\n\n test(\"kurtosis\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder](xs: List[X2[Int, A]])(\n implicit\n encEv: Encoder[(Int, A)],\n evCanBeDoubleA: CatalystCast[A, Double]\n ): Prop = univariatePropTemplate(xs)(\n kurtosis[A, X2[Int, A]],\n org.apache.spark.sql.functions.kurtosis\n )\n\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n }\n\n test(\"skewness\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder](xs: List[X2[Int, A]])(\n implicit\n encEv: Encoder[(Int, A)],\n evCanBeDoubleA: CatalystCast[A, Double]\n ): Prop = univariatePropTemplate(xs)(\n skewness[A, X2[Int, A]],\n org.apache.spark.sql.functions.skewness\n )\n\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n }\n\n test(\"stddev_pop\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder](xs: List[X2[Int, A]])(\n implicit\n encEv: Encoder[(Int, A)],\n evCanBeDoubleA: CatalystCast[A, Double]\n ): Prop = univariatePropTemplate(xs)(\n stddevPop[A, X2[Int, A]],\n org.apache.spark.sql.functions.stddev_pop\n )\n\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n }\n\n test(\"stddev_samp\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder](xs: List[X2[Int, A]])(\n implicit\n encEv: Encoder[(Int, A)],\n evCanBeDoubleA: CatalystCast[A, Double]\n ): Prop = univariatePropTemplate(xs)(\n stddevSamp[A, X2[Int, A]],\n org.apache.spark.sql.functions.stddev_samp\n )\n check(forAll(prop[Double] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/functions/DateTimeStringBehaviourUtils.scala", "content": "package frameless.functions\n\nimport org.apache.spark.sql.Row\n\nobject DateTimeStringBehaviourUtils {\n val nullHandler: Row => Option[Int] = _.get(0) match {\n case i: Int => Some(i)\n case _ => None\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/functions/DoubleBehaviourUtils.scala", "content": "package frameless\npackage functions\n\n/**\n * Some statistical functions in Spark can result in Double, Double.NaN or Null.\n * This tends to break ?= of the property based testing. Use the nanNullHandler function\n * here to alleviate this by mapping this NaN and Null to None. This will result in\n * functioning comparison again.\n */\nobject DoubleBehaviourUtils {\n // Mapping with this function is needed because spark uses Double.NaN for some semantics in the\n // correlation function. ?= for prop testing will use == underlying and will break because Double.NaN != Double.NaN\n private val nanHandler: Double => Option[Double] = value => if (!value.equals(Double.NaN)) Option(value) else None\n // Making sure that null => None and does not result in 0.0d because of row.getAs[Double]'s use of .asInstanceOf\n val nanNullHandler: Any => Option[Double] = {\n case null => None\n case d: Double => nanHandler(d)\n case _ => ???\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/functions/NonAggregateFunctionsTests.scala", "content": "package frameless\npackage functions\n\nimport java.io.File\nimport java.util.Base64\nimport java.nio.charset.StandardCharsets\n\nimport frameless.functions.nonAggregate._\nimport org.apache.commons.io.FileUtils\nimport org.apache.spark.sql.{Column, Encoder, SaveMode, functions => sparkFunctions}\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Arbitrary, Gen, Prop}\n\nimport scala.annotation.nowarn\n\nclass NonAggregateFunctionsTests extends TypedDatasetSuite {\n val testTempFiles = \"target/testoutput\"\n\n object NonNegativeGenerators {\n val doubleGen = for {\n s <- Gen.chooseNum(1, Int.MaxValue)\n e <- Gen.chooseNum(1, Int.MaxValue)\n res: Double = s.toDouble / e.toDouble\n } yield res\n\n val intGen: Gen[Int] = Gen.chooseNum(1, Int.MaxValue)\n val shortGen: Gen[Short] = Gen.chooseNum(1, Short.MaxValue)\n val longGen: Gen[Long] = Gen.chooseNum(1, Long.MaxValue)\n val byteGen: Gen[Byte] = Gen.chooseNum(1, Byte.MaxValue)\n }\n\n object NonNegativeArbitraryNumericValues {\n import NonNegativeGenerators._\n implicit val arbInt: Arbitrary[Int] = Arbitrary(intGen)\n implicit val arbDouble: Arbitrary[Double] = Arbitrary(doubleGen)\n implicit val arbLong: Arbitrary[Long] = Arbitrary(longGen)\n implicit val arbShort: Arbitrary[Short] = Arbitrary(shortGen)\n implicit val arbByte: Arbitrary[Byte] = Arbitrary(byteGen)\n }\n\n private val base64Encoder = Base64.getEncoder\n private def base64X1String(x1: X1[String]): X1[String] = {\n def base64(str: String): String = base64Encoder.encodeToString(str.getBytes(StandardCharsets.UTF_8))\n x1.copy(a = base64(x1.a))\n }\n\n override def afterAll(): Unit = {\n FileUtils.deleteDirectory(new File(testTempFiles))\n super.afterAll()\n }\n\n test(\"negate\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder](values: List[X1[A]])(\n implicit encX1:Encoder[X1[A]],\n catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, B]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.negate(cDS(\"a\")))\n .map(_.getAs[B](0))\n .collect()\n .toList\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(negate(col))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Byte, Byte] _))\n check(forAll(prop[Short, Short] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[BigDecimal, java.math.BigDecimal] _))\n }\n\n test(\"not\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[Boolean]], fromBase: Int, toBase: Int)(implicit encX1:Encoder[X1[Boolean]]) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.not(cDS(\"a\")))\n .map(_.getAs[Boolean](0))\n .collect()\n .toList\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(not(col))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"conv\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[String]], fromBase: Int, toBase: Int)(implicit encX1:Encoder[X1[String]]) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.conv(cDS(\"a\"), fromBase, toBase))\n .map(_.getAs[String](0))\n .collect()\n .toList\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(conv(col, fromBase, toBase))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"degrees\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.degrees(cDS(\"a\")))\n .map(_.getAs[Double](0))\n .collect()\n .toList\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(degrees(col))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Byte] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[BigDecimal] _))\n }\n\n def propBitShift[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder](typedDS: TypedDataset[X1[A]])\n (typedCol: TypedColumn[X1[A], B], sparkFunc: (Column,Int) => Column, numBits: Int): Prop = {\n val spark = session\n import spark.implicits._\n\n val resCompare = typedDS.dataset\n .select(sparkFunc($\"a\", numBits))\n .map(_.getAs[B](0))\n .collect()\n .toList\n\n val res = typedDS\n .select(typedCol)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n test(\"shiftRightUnsigned\") {\n val spark = session\n import spark.implicits._\n\n @nowarn // supress sparkFunctions.shiftRightUnsigned call which is used to maintain Spark 3.1.x backwards compat\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder]\n (values: List[X1[A]], numBits: Int)\n (implicit catalystBitShift: CatalystBitShift[A, B], encX1: Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propBitShift(typedDS)(shiftRightUnsigned(typedDS('a), numBits), sparkFunctions.shiftRightUnsigned, numBits)\n }\n\n check(forAll(prop[Byte, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[BigDecimal, Int] _))\n }\n\n test(\"shiftRight\") {\n val spark = session\n import spark.implicits._\n\n @nowarn // supress sparkFunctions.shiftRight call which is used to maintain Spark 3.1.x backwards compat\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder]\n (values: List[X1[A]], numBits: Int)\n (implicit catalystBitShift: CatalystBitShift[A, B], encX1: Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propBitShift(typedDS)(shiftRight(typedDS('a), numBits), sparkFunctions.shiftRight, numBits)\n }\n\n check(forAll(prop[Byte, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[BigDecimal, Int] _))\n }\n\n test(\"shiftLeft\") {\n val spark = session\n import spark.implicits._\n\n @nowarn // supress sparkFunctions.shiftLeft call which is used to maintain Spark 3.1.x backwards compat\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder]\n (values: List[X1[A]], numBits: Int)\n (implicit catalystBitShift: CatalystBitShift[A, B], encX1: Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propBitShift(typedDS)(shiftLeft(typedDS('a), numBits), sparkFunctions.shiftLeft, numBits)\n }\n\n check(forAll(prop[Byte, Int] _))\n check(forAll(prop[Short, Int] _))\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[BigDecimal, Int] _))\n }\n\n test(\"ceil\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder]\n (values: List[X1[A]])(\n implicit catalystAbsolute: CatalystRound[A, B], encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.ceil(cDS(\"a\")))\n .map(_.getAs[B](0))\n .collect()\n .toList.map{\n case bigDecimal : java.math.BigDecimal => bigDecimal.setScale(0)\n case other => other\n }.asInstanceOf[List[B]]\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(ceil(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[Short, Long] _))\n check(forAll(prop[Double, Long] _))\n check(forAll(prop[BigDecimal, java.math.BigDecimal] _))\n }\n\n test(\"sha2\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[Array[Byte]]])(implicit encX1: Encoder[X1[Array[Byte]]]) = {\n Seq(224, 256, 384, 512).map { numBits =>\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.sha2(cDS(\"a\"), numBits))\n .map(_.getAs[String](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(sha2(typedDS('a), numBits))\n .collect()\n .run()\n .toList\n res ?= resCompare\n }.reduce(_ && _)\n }\n\n check(forAll(prop _))\n }\n\n test(\"sha1\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[Array[Byte]]])(implicit encX1: Encoder[X1[Array[Byte]]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.sha1(cDS(\"a\")))\n .map(_.getAs[String](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(sha1(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"crc32\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[Array[Byte]]])(implicit encX1: Encoder[X1[Array[Byte]]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.crc32(cDS(\"a\")))\n .map(_.getAs[Long](0))\n .collect()\n .toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(crc32(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"floor\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder, B: TypedEncoder : Encoder]\n (values: List[X1[A]])(\n implicit catalystAbsolute: CatalystRound[A, B], encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.floor(cDS(\"a\")))\n .map(_.getAs[B](0))\n .collect()\n .toList.map{\n case bigDecimal : java.math.BigDecimal => bigDecimal.setScale(0)\n case other => other\n }.asInstanceOf[List[B]]\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(floor(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[Short, Long] _))\n check(forAll(prop[Double, Long] _))\n check(forAll(prop[BigDecimal, java.math.BigDecimal] _))\n }\n\n\n test(\"abs big decimal\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder: Encoder, B: TypedEncoder: Encoder]\n (values: List[X1[A]])\n (\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, B],\n encX1:Encoder[X1[A]]\n )= {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.abs(cDS(\"a\")))\n .map(_.getAs[B](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(\n abs(col)\n )\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[BigDecimal, java.math.BigDecimal] _))\n }\n\n test(\"abs\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder]\n (values: List[X1[A]])\n (\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, A],\n encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.abs(cDS(\"a\")))\n .map(_.getAs[A](0))\n .collect().toList\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(abs(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Double] _))\n }\n\n def propTrigonometric[A: CatalystNumeric: TypedEncoder : Encoder](typedDS: TypedDataset[X1[A]])\n (typedCol: TypedColumn[X1[A], Double], sparkFunc: Column => Column): Prop = {\n val spark = session\n import spark.implicits._\n\n val resCompare = typedDS.dataset\n .select(sparkFunc($\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(typedCol)\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n test(\"cos\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(cos(typedDS('a)), sparkFunctions.cos)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"cosh\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(cosh(typedDS('a)), sparkFunctions.cosh)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"acos\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(acos(typedDS('a)), sparkFunctions.acos)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n\n\n test(\"signum\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(signum(typedDS('a)), sparkFunctions.signum)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"sin\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(sin(typedDS('a)), sparkFunctions.sin)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"sinh\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(sinh(typedDS('a)), sparkFunctions.sinh)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"asin\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(asin(typedDS('a)), sparkFunctions.asin)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"tan\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(tan(typedDS('a)), sparkFunctions.tan)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"tanh\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])\n (implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n propTrigonometric(typedDS)(tanh(typedDS('a)), sparkFunctions.tanh)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n /*\n * Currently not all Collection types play nice with the Encoders.\n * This test needs to be readressed and Set readded to the Collection Typeclass once these issues are resolved.\n *\n * [[https://issues.apache.org/jira/browse/SPARK-18891]]\n * [[https://issues.apache.org/jira/browse/SPARK-21204]]\n */\n test(\"arrayContains\"){\n val spark = session\n import spark.implicits._\n\n val listLength = 10\n val idxs = Stream.continually(Range(0, listLength)).flatten.toIterator\n\n abstract class Nth[A, C[A]:CatalystCollection] {\n\n def nth(c:C[A], idx:Int):A\n }\n\n implicit def deriveListNth[A] : Nth[A, List] = new Nth[A, List] {\n override def nth(c: List[A], idx: Int): A = c(idx)\n }\n\n implicit def deriveSeqNth[A] : Nth[A, Seq] = new Nth[A, Seq] {\n override def nth(c: Seq[A], idx: Int): A = c(idx)\n }\n\n implicit def deriveVectorNth[A] : Nth[A, Vector] = new Nth[A, Vector] {\n override def nth(c: Vector[A], idx: Int): A = c(idx)\n }\n\n implicit def deriveArrayNth[A] : Nth[A, Array] = new Nth[A, Array] {\n override def nth(c: Array[A], idx: Int): A = c(idx)\n }\n\n\n def prop[C[_] : CatalystCollection]\n (\n values: C[Int],\n shouldBeIn:Boolean)\n (\n implicit nth:Nth[Int, C],\n encEv: Encoder[C[Int]],\n tEncEv: TypedEncoder[C[Int]]\n ) = {\n\n val contained = if (shouldBeIn) nth.nth(values, idxs.next) else -1\n\n val cDS = session.createDataset(List(values))\n val resCompare = cDS\n .select(sparkFunctions.array_contains(cDS(\"value\"), contained))\n .map(_.getAs[Boolean](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(List(X1(values)))\n val res = typedDS\n .select(arrayContains(typedDS('a), contained))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(\n forAll(\n Gen.listOfN(listLength, Gen.choose(0,100)),\n Gen.oneOf(true,false)\n )\n (prop[List])\n )\n\n /*check( Looks like there is no Typed Encoder for Seq type yet\n forAll(\n Gen.listOfN(listLength, Gen.choose(0,100)),\n Gen.oneOf(true,false)\n )\n (prop[Seq])\n )*/\n\n check(\n forAll(\n Gen.listOfN(listLength, Gen.choose(0,100)).map(_.toVector),\n Gen.oneOf(true,false)\n )\n (prop[Vector])\n )\n\n check(\n forAll(\n Gen.listOfN(listLength, Gen.choose(0,100)).map(_.toArray),\n Gen.oneOf(true,false)\n )\n (prop[Array])\n )\n }\n\n test(\"atan\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder]\n (na: A, values: List[X1[A]])(implicit encX1: Encoder[X1[A]]) = {\n val cDS = session.createDataset(X1(na) :: values)\n val resCompare = cDS\n .select(sparkFunctions.atan(cDS(\"a\")))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val typedDS = TypedDataset.create(cDS)\n val res = typedDS\n .select(atan(typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val aggrTyped = typedDS.agg(atan(\n frameless.functions.aggregate.first(typedDS('a)))\n ).firstOption().run().get\n\n val aggrSpark = cDS.select(\n sparkFunctions.atan(sparkFunctions.first(\"a\")).as[Double]\n ).first()\n\n (res ?= resCompare).&&(aggrTyped ?= aggrSpark)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"atan2\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder,\n B: CatalystNumeric : TypedEncoder : Encoder](na: X2[A, B], values: List[X2[A, B]])\n (implicit encEv: Encoder[X2[A,B]]) = {\n val cDS = session.createDataset(na +: values)\n val resCompare = cDS\n .select(sparkFunctions.atan2(cDS(\"a\"), cDS(\"b\")))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n\n val typedDS = TypedDataset.create(cDS)\n val res = typedDS\n .select(atan2(typedDS('a), typedDS('b)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val aggrTyped = typedDS.agg(atan2(\n frameless.functions.aggregate.first(typedDS('a)),\n frameless.functions.aggregate.first(typedDS('b)))\n ).firstOption().run().get\n\n val aggrSpark = cDS.select(\n sparkFunctions.atan2(sparkFunctions.first(\"a\"),sparkFunctions.first(\"b\")).as[Double]\n ).first()\n\n (res ?= resCompare).&&(aggrTyped ?= aggrSpark)\n }\n\n\n check(forAll(prop[Int, Long] _))\n check(forAll(prop[Long, Int] _))\n check(forAll(prop[Short, Byte] _))\n check(forAll(prop[BigDecimal, Double] _))\n check(forAll(prop[Byte, Int] _))\n check(forAll(prop[Double, Double] _))\n }\n\n test(\"atan2LitLeft\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder]\n (na: X1[A], value: List[X1[A]], lit:Double)(implicit encX1:Encoder[X1[A]]) = {\n val cDS = session.createDataset(na +: value)\n val resCompare = cDS\n .select(sparkFunctions.atan2(lit, cDS(\"a\")))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n\n val typedDS = TypedDataset.create(cDS)\n val res = typedDS\n .select(atan2(lit, typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val aggrTyped = typedDS.agg(atan2(\n lit,\n frameless.functions.aggregate.first(typedDS('a)))\n ).firstOption().run().get\n\n val aggrSpark = cDS.select(\n sparkFunctions.atan2(lit, sparkFunctions.first(\"a\")).as[Double]\n ).first()\n\n (res ?= resCompare).&&(aggrTyped ?= aggrSpark)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"atan2LitRight\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder]\n (na: X1[A], value: List[X1[A]], lit:Double)(implicit encX1:Encoder[X1[A]]) = {\n val cDS = session.createDataset(na +: value)\n val resCompare = cDS\n .select(sparkFunctions.atan2(cDS(\"a\"), lit))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n\n val typedDS = TypedDataset.create(cDS)\n val res = typedDS\n .select(atan2(typedDS('a), lit))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val aggrTyped = typedDS.agg(atan2(\n frameless.functions.aggregate.first(typedDS('a)),\n lit)\n ).firstOption().run().get\n\n val aggrSpark = cDS.select(\n sparkFunctions.atan2(sparkFunctions.first(\"a\"), lit).as[Double]\n ).first()\n\n (res ?= resCompare).&&(aggrTyped ?= aggrSpark)\n }\n\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n def mathProp[A: CatalystNumeric: TypedEncoder : Encoder](typedDS: TypedDataset[X1[A]])(\n typedCol: TypedColumn[X1[A], Double], sparkFunc: Column => Column\n ): Prop = {\n val spark = session\n import spark.implicits._\n\n val resCompare = typedDS.dataset\n .select(sparkFunc($\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(typedCol)\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n test(\"sqrt\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(sqrt(typedDS('a)), sparkFunctions.sqrt)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"crbt\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(cbrt(typedDS('a)), sparkFunctions.cbrt)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"exp\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(exp(typedDS('a)), sparkFunctions.exp)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[BigDecimal] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"md5\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]]): Prop = {\n val spark = session\n import spark.implicits._\n\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.md5($\"a\"))\n .map(_.getAs[String](0))\n .collect().toList\n\n val res = typedDS\n .select(md5(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[String] _))\n }\n\n test(\"factorial\") {\n val spark = session\n\n def prop(values: List[X1[Long]]): Prop = {\n val spark = session\n import spark.implicits._\n\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.factorial($\"a\"))\n .map(_.getAs[Long](0))\n .collect().toList\n\n val res = typedDS\n .select(factorial(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"round\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]])(\n implicit catalystNumericWithJavaBigDecimal: CatalystNumericWithJavaBigDecimal[A, A],\n encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.round(cDS(\"a\")))\n .map(_.getAs[A](0))\n .collect().toList\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(round(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"round big decimal\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder: Encoder](values: List[X1[A]])(\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, java.math.BigDecimal],\n encX1:Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.round(cDS(\"a\")))\n .map(_.getAs[java.math.BigDecimal](0))\n .collect()\n .toList.map(_.setScale(0))\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(round(col))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[BigDecimal] _))\n }\n\n test(\"round with scale\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]])(\n implicit catalystNumericWithJavaBigDecimal: CatalystNumericWithJavaBigDecimal[A, A],\n encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.round(cDS(\"a\"), 1))\n .map(_.getAs[A](0))\n .collect().toList\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(round(typedDS('a), 1))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"round big decimal with scale\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder: Encoder](values: List[X1[A]])(\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, java.math.BigDecimal],\n encX1:Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.round(cDS(\"a\"), 0))\n .map(_.getAs[java.math.BigDecimal](0))\n .collect()\n .toList.map(_.setScale(0))\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(round(col, 0))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[BigDecimal] _))\n }\n\n test(\"bround\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]])(\n implicit catalystNumericWithJavaBigDecimal: CatalystNumericWithJavaBigDecimal[A, A],\n encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.bround(cDS(\"a\")))\n .map(_.getAs[A](0))\n .collect().toList\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(bround(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"bround big decimal\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder: Encoder](values: List[X1[A]])(\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, java.math.BigDecimal],\n encX1:Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.bround(cDS(\"a\")))\n .map(_.getAs[java.math.BigDecimal](0))\n .collect()\n .toList.map(_.setScale(0))\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(bround(col))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[BigDecimal] _))\n }\n\n test(\"bround with scale\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder : Encoder](values: List[X1[A]])(\n implicit catalystNumericWithJavaBigDecimal: CatalystNumericWithJavaBigDecimal[A, A],\n encX1: Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.bround(cDS(\"a\"), 1))\n .map(_.getAs[A](0))\n .collect().toList\n\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(bround(typedDS('a), 1))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"bround big decimal with scale\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: TypedEncoder: Encoder](values: List[X1[A]])(\n implicit catalystAbsolute: CatalystNumericWithJavaBigDecimal[A, java.math.BigDecimal],\n encX1:Encoder[X1[A]]\n ) = {\n val cDS = session.createDataset(values)\n\n val resCompare = cDS\n .select(sparkFunctions.bround(cDS(\"a\"), 0))\n .map(_.getAs[java.math.BigDecimal](0))\n .collect()\n .toList.map(_.setScale(0))\n\n val typedDS = TypedDataset.create(values)\n val col = typedDS('a)\n val res = typedDS\n .select(bround(col, 0))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[BigDecimal] _))\n }\n\n test(\"log with base\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X1[A]],\n base: Double\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.log(base, $\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(log(base, typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"log\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(log(typedDS('a)), sparkFunctions.log)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"log2\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(log2(typedDS('a)), sparkFunctions.log2)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"log1p\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(log1p(typedDS('a)), sparkFunctions.log1p)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"log10\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric : TypedEncoder : Encoder](values: List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val typedDS = TypedDataset.create(values)\n mathProp(typedDS)(log10(typedDS('a)), sparkFunctions.log10)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"base64\") {\n val spark = session\n import spark.implicits._\n\n def prop(values:List[X1[Array[Byte]]])(implicit encX1:Encoder[X1[Array[Byte]]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.base64(cDS(\"a\")))\n .map(_.getAs[String](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(base64(typedDS('a)))\n .collect()\n .run()\n .toList\n\n val backAndForth = typedDS\n .select(base64(unbase64(base64(typedDS('a)))))\n .collect()\n .run()\n .toList\n\n (res ?= resCompare) && (res ?= backAndForth)\n }\n\n check(forAll(prop _))\n }\n\n test(\"hypot with double\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X1[A]],\n base: Double\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.hypot(base, $\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res2 = typedDS\n .select(hypot(typedDS('a), base))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val res = typedDS\n .select(hypot(base, typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n (res ?= resCompare) && (res2 ?= resCompare)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"hypot with two columns\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X2[A, A]]\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.hypot($\"b\", $\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(hypot(typedDS('b), typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"pow with double\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X1[A]],\n base: Double\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.pow(base, $\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(pow(base, typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n val resCompare2 = typedDS.dataset\n .select(sparkFunctions.pow($\"a\", base))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res2 = typedDS\n .select(pow(typedDS('a), base))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n (res ?= resCompare) && (res2 ?= resCompare2)\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"pow with two columns\") {\n val spark = session\n import spark.implicits._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X2[A, A]]\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.pow($\"b\", $\"a\"))\n .map(_.getAs[Double](0))\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect().toList\n\n val res = typedDS\n .select(pow(typedDS('b), typedDS('a)))\n .deserialized\n .map(DoubleBehaviourUtils.nanNullHandler)\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"pmod\") {\n val spark = session\n import spark.implicits._\n import NonNegativeArbitraryNumericValues._\n\n def prop[A: CatalystNumeric: TypedEncoder : Encoder](\n values: List[X2[A, A]]\n ): Prop = {\n val spark = session\n import spark.implicits._\n val typedDS = TypedDataset.create(values)\n\n val resCompare = typedDS.dataset\n .select(sparkFunctions.pmod($\"b\", $\"a\"))\n .map(_.getAs[A](0))\n .collect().toList\n\n val res = typedDS\n .select(pmod(typedDS('b), typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Double] _))\n }\n\n test(\"unbase64\") {\n val spark = session\n import spark.implicits._\n\n def prop(values: List[X1[String]])(implicit encX1: Encoder[X1[String]]) = {\n val valuesBase64 = values.map(base64X1String)\n val cDS = session.createDataset(valuesBase64)\n val resCompare = cDS\n .select(sparkFunctions.unbase64(cDS(\"a\")))\n .map(_.getAs[Array[Byte]](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(valuesBase64)\n val res = typedDS\n .select(unbase64(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res.map(_.toList) ?= resCompare.map(_.toList)\n }\n\n check(forAll(prop _))\n }\n\n test(\"bin\"){\n val spark = session\n import spark.implicits._\n\n def prop(values:List[X1[Long]])(implicit encX1:Encoder[X1[Long]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.bin(cDS(\"a\")))\n .map(_.getAs[String](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(bin(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop _))\n }\n\n test(\"bitwiseNOT\"){\n val spark = session\n import spark.implicits._\n\n @nowarn // supress sparkFunctions.bitwiseNOT call which is used to maintain Spark 3.1.x backwards compat\n def prop[A: CatalystBitwise : TypedEncoder : Encoder]\n (values:List[X1[A]])(implicit encX1:Encoder[X1[A]]) = {\n val cDS = session.createDataset(values)\n val resCompare = cDS\n .select(sparkFunctions.bitwiseNOT(cDS(\"a\")))\n .map(_.getAs[A](0))\n .collect().toList\n\n val typedDS = TypedDataset.create(values)\n val res = typedDS\n .select(bitwiseNOT(typedDS('a)))\n .collect()\n .run()\n .toList\n\n res ?= resCompare\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Int] _))\n }\n\n test(\"inputFileName\") {\n val spark = session\n import spark.implicits._\n\n def prop[A : TypedEncoder](\n toFile1: List[X1[A]],\n toFile2: List[X1[A]],\n inMem: List[X1[A]]\n )(implicit x2Gen: Encoder[X2[A, String]], x3Gen: Encoder[X3[A, String, String]]) = {\n\n val file1Path = testTempFiles + \"/file1\"\n val file2Path = testTempFiles + \"/file2\"\n\n val toFile1WithName = toFile1.map(x => X2(x.a, \"file1\"))\n val toFile2WithName = toFile2.map(x => X2(x.a, \"file2\"))\n val inMemWithName = inMem.map(x => X2(x.a, \"\"))\n\n toFile1WithName.toDS().write.mode(SaveMode.Overwrite).parquet(file1Path)\n toFile2WithName.toDS().write.mode(SaveMode.Overwrite).parquet(file2Path)\n\n val readBackIn1 = spark.read.parquet(file1Path).as[X2[A, String]]\n val readBackIn2 = spark.read.parquet(file2Path).as[X2[A, String]]\n\n val ds1 = TypedDataset.create(readBackIn1)\n val ds2 = TypedDataset.create(readBackIn2)\n val ds3 = TypedDataset.create(inMemWithName)\n\n val unioned = ds1.union(ds2).union(ds3)\n\n val withFileName = unioned.withColumn[X3[A, String, String]](inputFileName[X2[A, String]]())\n .collect()\n .run()\n .toVector\n\n val grouped = withFileName.groupBy(_.b).mapValues(_.map(_.c).toSet)\n\n grouped.foldLeft(passed) { (p, g) =>\n p && secure { g._1 match {\n case \"\" => g._2.head == \"\" //Empty string if didn't come from file\n case f => g._2.forall(_.contains(f))\n }}}\n }\n\n check(forAll(prop[String] _))\n }\n\n test(\"monotonic id\") {\n val spark = session\n import spark.implicits._\n\n def prop[A : TypedEncoder](xs: List[X1[A]])(implicit x2en: Encoder[X2[A, Long]]) = {\n val ds = TypedDataset.create(xs)\n\n val result = ds.withColumn[X2[A, Long]](monotonicallyIncreasingId())\n .collect()\n .run()\n .toVector\n\n val ids = result.map(_.b)\n (ids.toSet.size ?= ids.length) &&\n (ids.sorted ?= ids)\n }\n\n check(forAll(prop[String] _))\n }\n\n test(\"when\") {\n val spark = session\n import spark.implicits._\n\n def prop[A : TypedEncoder : Encoder]\n (condition1: Boolean, condition2: Boolean, value1: A, value2: A, otherwise: A) = {\n val ds = TypedDataset.create(X5(condition1, condition2, value1, value2, otherwise) :: Nil)\n\n val untypedWhen = ds.toDF()\n .select(\n sparkFunctions.when(sparkFunctions.col(\"a\"), sparkFunctions.col(\"c\"))\n .when(sparkFunctions.col(\"b\"), sparkFunctions.col(\"d\"))\n .otherwise(sparkFunctions.col(\"e\"))\n )\n .as[A]\n .collect()\n .toList\n\n val typedWhen = ds\n .select(\n when(ds('a), ds('c))\n .when(ds('b), ds('d))\n .otherwise(ds('e))\n )\n .collect()\n .run()\n .toList\n\n typedWhen ?= untypedWhen\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Short] _))\n check(forAll(prop[Byte] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Option[Int]] _))\n }\n\n test(\"ascii\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.ascii($\"a\"))\n .map(_.getAs[Int](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(ascii(ds('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"concat\") {\n val spark = session\n import spark.implicits._\n\n val pairs = for {\n y <- Gen.alphaStr\n x <- Gen.nonEmptyListOf(X2(y, y))\n } yield x\n\n check(forAll(pairs) { values: List[X2[String, String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.concat($\"a\", $\"b\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(concat(ds('a), ds('b)))\n .collect()\n .run()\n .toVector\n\n (typed ?= sparkResult).&&(typed ?= values.map(x => s\"${x.a}${x.b}\").toVector)\n })\n }\n\n test(\"concat for TypedAggregate\") {\n val spark = session\n import frameless.functions.aggregate._\n import spark.implicits._\n val pairs = for {\n y <- Gen.alphaStr\n x <- Gen.nonEmptyListOf(X2(y, y))\n } yield x\n\n check(forAll(pairs) { values: List[X2[String, String]] =>\n val ds = TypedDataset.create(values)\n val td = ds.agg(concat(first(ds('a)),first(ds('b)))).collect().run().toVector\n val spark = ds.dataset.select(sparkFunctions.concat(\n sparkFunctions.first($\"a\").as[String],\n sparkFunctions.first($\"b\").as[String])).as[String].collect().toVector\n td ?= spark\n })\n }\n\n test(\"concat_ws\") {\n val spark = session\n import spark.implicits._\n\n val pairs = for {\n y <- Gen.alphaStr\n x <- Gen.nonEmptyListOf(X2(y, y))\n } yield x\n\n check(forAll(pairs) { values: List[X2[String, String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.concat_ws(\",\", $\"a\", $\"b\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(concatWs(\",\", ds('a), ds('b)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"concat_ws for TypedAggregate\") {\n val spark = session\n import frameless.functions.aggregate._\n import spark.implicits._\n val pairs = for {\n y <- Gen.alphaStr\n x <- Gen.listOfN(10, X2(y, y))\n } yield x\n\n check(forAll(pairs) { values: List[X2[String, String]] =>\n val ds = TypedDataset.create(values)\n val td = ds.agg(concatWs(\",\",first(ds('a)),first(ds('b)), last(ds('b)))).collect().run().toVector\n val spark = ds.dataset.select(sparkFunctions.concat_ws(\",\",\n sparkFunctions.first($\"a\").as[String],\n sparkFunctions.first($\"b\").as[String],\n sparkFunctions.last($\"b\").as[String])).as[String].collect().toVector\n td ?= spark\n })\n }\n\n test(\"instr\") {\n val spark = session\n import spark.implicits._\n check(forAll(Gen.nonEmptyListOf(Gen.alphaStr)) { values: List[String] =>\n val ds = TypedDataset.create(values.map(x => X1(x + values.head)))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.instr($\"a\", values.head))\n .map(_.getAs[Int](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(instr(ds('a), values.head))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"length\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.length($\"a\"))\n .map(_.getAs[Int](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(length(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n (typed ?= sparkResult).&&(values.map(_.a.length).toVector ?= typed)\n })\n }\n\n test(\"levenshtein\") {\n val spark = session\n import spark.implicits._\n check(forAll { (na: X1[String], values: List[X1[String]]) =>\n val ds = TypedDataset.create(na +: values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.levenshtein($\"a\", sparkFunctions.concat($\"a\",sparkFunctions.lit(\"Hello\"))))\n .map(_.getAs[Int](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(levenshtein(ds('a), concat(ds('a),lit(\"Hello\"))))\n .collect()\n .run()\n .toVector\n\n val cDS = ds.dataset\n val aggrTyped = ds.agg(\n levenshtein(frameless.functions.aggregate.first(ds('a)), litAggr(\"Hello\"))\n ).firstOption().run().get\n\n val aggrSpark = cDS.select(\n sparkFunctions.levenshtein(sparkFunctions.first(\"a\"), sparkFunctions.lit(\"Hello\")).as[Int]\n ).first()\n\n (typed ?= sparkResult).&&(aggrTyped ?= aggrSpark)\n })\n }\n\n test(\"regexp_replace\") {\n val spark = session\n import spark.implicits._\n check(forAll { (values: List[X1[String]], n: Int) =>\n val ds = TypedDataset.create(values.map(x => X1(s\"$n${x.a}-$n$n\")))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.regexp_replace($\"a\", \"\\\\d+\", \"n\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(regexpReplace(ds[String]('a), \"\\\\d+\".r, \"n\"))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"reverse\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.reverse($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(reverse(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n (typed ?= sparkResult).&&(values.map(_.a.reverse).toVector ?= typed)\n })\n }\n\n test(\"rpad\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.rpad($\"a\", 5, \"hello\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(rpad(ds[String]('a), 5, \"hello\"))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"lpad\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.lpad($\"a\", 5, \"hello\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(lpad(ds[String]('a), 5, \"hello\"))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"rtrim\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values.map(x => X1(s\" ${x.a} \")))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.rtrim($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(rtrim(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"ltrim\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values.map(x => X1(s\" ${x.a} \")))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.ltrim($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(ltrim(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"substring\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values)\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.substring($\"a\", 5, 3))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(substring(ds[String]('a), 5, 3))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"trim\") {\n val spark = session\n import spark.implicits._\n check(forAll { values: List[X1[String]] =>\n val ds = TypedDataset.create(values.map(x => X1(s\" ${x.a} \")))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.trim($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(trim(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"upper\") {\n val spark = session\n import spark.implicits._\n check(forAll(Gen.listOf(Gen.alphaStr)) { values: List[String] =>\n val ds = TypedDataset.create(values.map(X1(_)))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.upper($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(upper(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"lower\") {\n val spark = session\n import spark.implicits._\n check(forAll(Gen.listOf(Gen.alphaStr)) { values: List[String] =>\n val ds = TypedDataset.create(values.map(X1(_)))\n\n val sparkResult = ds.toDF()\n .select(sparkFunctions.lower($\"a\"))\n .map(_.getAs[String](0))\n .collect()\n .toVector\n\n val typed = ds\n .select(lower(ds[String]('a)))\n .collect()\n .run()\n .toVector\n\n typed ?= sparkResult\n })\n }\n\n test(\"Empty vararg tests\") {\n def prop[A : TypedEncoder, B: TypedEncoder](data: Vector[X2[A, B]]) = {\n val ds = TypedDataset.create(data)\n val frameless = ds.select(ds('a), concat(), ds('b), concatWs(\":\")).collect().run().toVector\n val framelessAggr = ds.agg(concat(), concatWs(\"x\"), litAggr(2)).collect().run().toVector\n val scala = data.map(x => (x.a, \"\", x.b, \"\"))\n val scalaAggr = Vector((\"\", \"\", 2))\n (frameless ?= scala).&&(framelessAggr ?= scalaAggr)\n }\n\n check(forAll(prop[Long, Long] _))\n check(forAll(prop[Option[Boolean], Long] _))\n }\n\n def dateTimeStringProp(typedDS: TypedDataset[X1[String]])\n (typedCol: TypedColumn[X1[String], Option[Int]], sparkFunc: Column => Column): Prop = {\n val spark = session\n import spark.implicits._\n\n val sparkResult = typedDS.dataset\n .select(sparkFunc($\"a\"))\n .map(DateTimeStringBehaviourUtils.nullHandler)\n .collect()\n .toList\n\n val typed = typedDS\n .select(typedCol)\n .collect()\n .run()\n .toList\n\n typed ?= sparkResult\n }\n\n test(\"year\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(year(ds[String]('a)), sparkFunctions.year)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"quarter\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(quarter(ds[String]('a)), sparkFunctions.quarter)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"month\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(month(ds[String]('a)), sparkFunctions.month)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"dayofweek\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(dayofweek(ds[String]('a)), sparkFunctions.dayofweek)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"dayofmonth\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(dayofmonth(ds[String]('a)), sparkFunctions.dayofmonth)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"dayofyear\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(dayofyear(ds[String]('a)), sparkFunctions.dayofyear)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"hour\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(hour(ds[String]('a)), sparkFunctions.hour)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"minute\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(minute(ds[String]('a)), sparkFunctions.minute)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"second\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(second(ds[String]('a)), sparkFunctions.second)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n\n test(\"weekofyear\") {\n val spark = session\n import spark.implicits._\n\n def prop(data: List[X1[String]])(implicit E: Encoder[Option[Int]]): Prop = {\n val ds = TypedDataset.create(data)\n dateTimeStringProp(ds)(weekofyear(ds[String]('a)), sparkFunctions.weekofyear)\n }\n\n check(forAll(dateTimeStringGen)(data => prop(data.map(X1.apply))))\n check(forAll(prop _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/functions/UdfTests.scala", "content": "package frameless\npackage functions\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass UdfTests extends TypedDatasetSuite {\n\n test(\"one argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder](data: Vector[X1[A]], f1: A => B): Prop = {\n val dataset: TypedDataset[X1[A]] = TypedDataset.create(data)\n val u1 = udf[X1[A], A, B](f1)\n val u2 = dataset.makeUDF(f1)\n val A = dataset.col[A]('a)\n\n // filter forces whole codegen\n val codegen = dataset.deserialized.filter((_:X1[A]) => true).select(u1(A)).collect().run().toVector\n\n // otherwise it uses local relation\n val local = dataset.select(u2(A)).collect().run().toVector\n\n val d = data.map(x => f1(x.a))\n\n (codegen ?= d) && (local ?= d)\n }\n\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[String, String] _))\n check(forAll(prop[Option[Int], Option[Int]] _))\n check(forAll(prop[X1[Int], X1[Int]] _))\n check(forAll(prop[X1[Option[Int]], X1[Option[Int]]] _))\n\n // TODO doesn't work for the same reason as `collect`\n // check(forAll(prop[X1[Option[X1[Int]]], X1[Option[X1[Option[Int]]]]] _))\n\n check(forAll(prop[Option[Vector[String]], Option[Vector[String]]] _))\n\n def prop2[A: TypedEncoder, B: TypedEncoder](f: A => B)(a: A): Prop = prop(Vector(X1(a)), f)\n\n check(forAll(prop2[Int, Option[Int]](x => if (x % 2 == 0) Some(x) else None) _))\n check(forAll(prop2[Option[Int], Int](x => x getOrElse 0) _))\n }\n\n test(\"multiple one argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder]\n (data: Vector[X3[A, B, C]], f1: A => A, f2: B => B, f3: C => C): Prop = {\n val dataset = TypedDataset.create(data)\n val u11 = udf[X3[A, B, C], A, A](f1)\n val u21 = udf[X3[A, B, C], B, B](f2)\n val u31 = udf[X3[A, B, C], C, C](f3)\n val u12 = dataset.makeUDF(f1)\n val u22 = dataset.makeUDF(f2)\n val u32 = dataset.makeUDF(f3)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val dataset21 = dataset.select(u11(A), u21(B), u31(C)).collect().run().toVector\n val dataset22 = dataset.select(u12(A), u22(B), u32(C)).collect().run().toVector\n val d = data.map(x => (f1(x.a), f2(x.b), f3(x.c)))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int] _))\n check(forAll(prop[String, Int, Int] _))\n check(forAll(prop[X3[Int, String, Boolean], Int, Int] _))\n check(forAll(prop[X3U[Int, String, Boolean], Int, Int] _))\n }\n\n test(\"two argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder]\n (data: Vector[X3[A, B, C]], f1: (A, B) => C): Prop = {\n val dataset = TypedDataset.create(data)\n val u1 = udf[X3[A, B, C], A, B, C](f1)\n val u2 = dataset.makeUDF(f1)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val dataset21 = dataset.select(u1(A, B)).collect().run().toVector\n val dataset22 = dataset.select(u2(A, B)).collect().run().toVector\n val d = data.map(x => f1(x.a, x.b))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int] _))\n check(forAll(prop[String, Int, Int] _))\n }\n\n test(\"multiple two argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder]\n (data: Vector[X3[A, B, C]], f1: (A, B) => C, f2: (B, C) => A): Prop = {\n val dataset = TypedDataset.create(data)\n val u11 = udf[X3[A, B, C], A, B, C](f1)\n val u12 = dataset.makeUDF(f1)\n val u21 = udf[X3[A, B, C], B, C, A](f2)\n val u22 = dataset.makeUDF(f2)\n\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val dataset21 = dataset.select(u11(A, B), u21(B, C)).collect().run().toVector\n val dataset22 = dataset.select(u12(A, B), u22(B, C)).collect().run().toVector\n val d = data.map(x => (f1(x.a, x.b), f2(x.b, x.c)))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int] _))\n check(forAll(prop[String, Int, Int] _))\n }\n\n test(\"three argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder]\n (data: Vector[X3[A, B, C]], f: (A, B, C) => C): Prop = {\n val dataset = TypedDataset.create(data)\n val u1 = udf[X3[A, B, C], A, B, C, C](f)\n val u2 = dataset.makeUDF(f)\n\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val dataset21 = dataset.select(u1(A, B, C)).collect().run().toVector\n val dataset22 = dataset.select(u2(A, B, C)).collect().run().toVector\n val d = data.map(x => f(x.a, x.b, x.c))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int] _))\n check(forAll(prop[String, Int, Int] _))\n }\n\n test(\"four argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder, D: TypedEncoder]\n (data: Vector[X4[A, B, C, D]], f: (A, B, C, D) => C): Prop = {\n val dataset = TypedDataset.create(data)\n val u1 = udf[X4[A, B, C, D], A, B, C, D, C](f)\n val u2 = dataset.makeUDF(f)\n\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n\n val dataset21 = dataset.select(u1(A, B, C, D)).collect().run().toVector\n val dataset22 = dataset.select(u2(A, B, C, D)).collect().run().toVector\n val d = data.map(x => f(x.a, x.b, x.c, x.d))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int, Int] _))\n check(forAll(prop[String, Int, Int, String] _))\n check(forAll(prop[String, String, String, String] _))\n check(forAll(prop[String, Long, String, String] _))\n check(forAll(prop[String, Boolean, Boolean, String] _))\n }\n\n test(\"five argument udf\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder, D: TypedEncoder, E: TypedEncoder]\n (data: Vector[X5[A, B, C, D, E]], f: (A, B, C, D, E) => C): Prop = {\n val dataset = TypedDataset.create(data)\n val u1 = udf[X5[A, B, C, D, E], A, B, C, D, E, C](f)\n val u2 = dataset.makeUDF(f)\n\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n val E = dataset.col[E]('e)\n\n val dataset21 = dataset.select(u1(A, B, C, D, E)).collect().run().toVector\n val dataset22 = dataset.select(u2(A, B, C, D, E)).collect().run().toVector\n val d = data.map(x => f(x.a, x.b, x.c, x.d, x.e))\n\n (dataset21 ?= d) && (dataset22 ?= d)\n }\n\n check(forAll(prop[Int, Int, Int, Int, Int] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/functions/UnaryFunctionsTest.scala", "content": "package frameless\npackage functions\n\nimport org.scalacheck.{ Arbitrary, Prop }\nimport org.scalacheck.Prop._\nimport scala.collection.SeqLike\n\nimport scala.math.Ordering\nimport scala.reflect.ClassTag\n\nclass UnaryFunctionsTest extends TypedDatasetSuite {\n test(\"size tests\") {\n def prop[F[X] <: Traversable[X] : CatalystSizableCollection, A](xs: List[X1[F[A]]])(implicit arb: Arbitrary[F[A]], enc: TypedEncoder[F[A]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(size(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.size).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Vector, Long] _))\n check(forAll(prop[List, Long] _))\n check(forAll(prop[Vector, Char] _))\n check(forAll(prop[List, Char] _))\n check(forAll(prop[Vector, X2[Int, Option[Long]]] _))\n check(forAll(prop[List, X2[Int, Option[Long]]] _))\n }\n\n test(\"size on array test\") {\n def prop[A: TypedEncoder: ClassTag](xs: List[X1[Array[A]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(size(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.size).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[String] _))\n check(forAll(prop[X2[Int, Option[Long]]] _))\n }\n\n test(\"size on Map\") {\n def prop[A](xs: List[X1[Map[A, A]]])(implicit arb: Arbitrary[Map[A, A]], enc: TypedEncoder[Map[A, A]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(size(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.size).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[Char] _))\n }\n\n test(\"sort in ascending order\") {\n def prop[F[X] <: SeqLike[X, F[X]] : CatalystSortableCollection, A: Ordering](xs: List[X1[F[A]]])(implicit enc: TypedEncoder[F[A]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(sortAscending(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.sorted).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Vector, Long] _))\n check(forAll(prop[Vector, Int] _))\n check(forAll(prop[Vector, Char] _))\n check(forAll(prop[Vector, String] _))\n check(forAll(prop[List, Long] _))\n check(forAll(prop[List, Int] _))\n check(forAll(prop[List, Char] _))\n check(forAll(prop[List, String] _))\n }\n\n test(\"sort in descending order\") {\n def prop[F[X] <: SeqLike[X, F[X]] : CatalystSortableCollection, A: Ordering](xs: List[X1[F[A]]])(implicit enc: TypedEncoder[F[A]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(sortDescending(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.sorted.reverse).toVector\n\n framelessResults ?= scalaResults\n }\n\n check(forAll(prop[Vector, Long] _))\n check(forAll(prop[Vector, Int] _))\n check(forAll(prop[Vector, Char] _))\n check(forAll(prop[Vector, String] _))\n check(forAll(prop[List, Long] _))\n check(forAll(prop[List, Int] _))\n check(forAll(prop[List, Char] _))\n check(forAll(prop[List, String] _))\n }\n\n test(\"sort on array test: ascending order\") {\n def prop[A: TypedEncoder : Ordering : ClassTag](xs: List[X1[Array[A]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(sortAscending(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.sorted).toVector\n\n Prop {\n framelessResults\n .zip(scalaResults)\n .forall {\n case (a, b) => a sameElements b\n }\n }\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n\n test(\"sort on array test: descending order\") {\n def prop[A: TypedEncoder : Ordering : ClassTag](xs: List[X1[Array[A]]]): Prop = {\n val tds = TypedDataset.create(xs)\n\n val framelessResults = tds.select(sortDescending(tds('a))).collect().run().toVector\n val scalaResults = xs.map(x => x.a.sorted.reverse).toVector\n\n Prop {\n framelessResults\n .zip(scalaResults)\n .forall {\n case (a, b) => a sameElements b\n }\n }\n }\n\n check(forAll(prop[Long] _))\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/ColumnTypesTest.scala", "content": "package frameless\npackage ops\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop.forAll\nimport shapeless.HNil\nimport shapeless.::\n\nclass ColumnTypesTest extends TypedDatasetSuite {\n test(\"test summoning\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder, D: TypedEncoder](data: Vector[X4[A, B, C, D]]): Prop = {\n val d: TypedDataset[X4[A, B, C, D]] = TypedDataset.create(data)\n val hlist = d('a) :: d('b) :: d('c) :: d('d) :: HNil\n\n type TC[N] = TypedColumn[X4[A,B,C,D], N]\n\n type IN = TC[A] :: TC[B] :: TC[C] :: TC[D] :: HNil\n type OUT = A :: B :: C :: D :: HNil\n\n implicitly[ColumnTypes.Aux[X4[A,B,C,D], IN, OUT]]\n Prop.passed // successful compilation implies test correctness\n }\n\n check(forAll(prop[Int, String, X1[String], Boolean] _))\n check(forAll(prop[Vector[Int], Vector[Vector[String]], X1[String], Option[String]] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/CubeTests.scala", "content": "package frameless\npackage ops\n\nimport frameless.functions.aggregate._\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass CubeTests extends TypedDatasetSuite {\n\n test(\"cube('a).agg(count())\") {\n def prop[A: TypedEncoder : Ordering, Out: TypedEncoder : Numeric]\n (data: List[X1[A]])(implicit summable: CatalystSummable[A, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.cube(A).agg(count()).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.cube(\"a\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Long](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n\n test(\"cube('a, 'b).agg(count())\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder, Out: TypedEncoder : Numeric]\n (data: List[X2[A, B]])(implicit summable: CatalystSummable[B, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val received = dataset.cube(A, B).agg(count()).collect().run().toVector.sortBy(_._3)\n val expected = dataset.dataset.cube(\"a\", \"b\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[Long](2))).sortBy(_._3)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"cube('a).agg(sum('b)\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder, Out: TypedEncoder : Numeric]\n (data: List[X2[A, B]])(implicit summable: CatalystSummable[B, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val received = dataset.cube(A).agg(sum(B)).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.cube(\"a\").sum(\"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Out](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"cube('a).mapGroups('a, sum('b))\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder : Numeric]\n (data: List[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.cube(A)\n .deserialized.mapGroups { case (a, xs) => (a, xs.map(_.b).sum) }\n .collect().run().toVector.sortBy(_._1)\n val expected = data.groupBy(_.a).mapValues(_.map(_.b).sum).toVector.sortBy(_._1)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n\n test(\"cube('a).agg(sum('b), sum('c)) to cube('a).agg(sum('a), sum('b), sum('a), sum('b), sum('a))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder,\n C: TypedEncoder,\n OutB: TypedEncoder : Numeric,\n OutC: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]])(\n implicit\n summableB: CatalystSummable[B, OutB],\n summableC: CatalystSummable[C, OutC]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumBC = dataset\n .cube(A)\n .agg(sum(B), sum(C))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBC = dataset.dataset.cube(\"a\").sum(\"b\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2)))\n .sortBy(_._1)\n\n val framelessSumBCB = dataset\n .cube(A)\n .agg(sum(B), sum(C), sum(B))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCB = dataset.dataset.cube(\"a\").sum(\"b\", \"c\", \"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3)))\n .sortBy(_._1)\n\n val framelessSumBCBC = dataset\n .cube(A)\n .agg(sum(B), sum(C), sum(B), sum(C))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCBC = dataset.dataset.cube(\"a\").sum(\"b\", \"c\", \"b\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3), row.getAs[OutC](4)))\n .sortBy(_._1)\n\n val framelessSumBCBCB = dataset\n .cube(A)\n .agg(sum(B), sum(C), sum(B), sum(C), sum(B))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCBCB = dataset.dataset.cube(\"a\").sum(\"b\", \"c\", \"b\", \"c\", \"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3), row.getAs[OutC](4), row.getAs[OutB](5)))\n .sortBy(_._1)\n\n (framelessSumBC ?= sparkSumBC)\n .&&(framelessSumBCB ?= sparkSumBCB)\n .&&(framelessSumBCBC ?= sparkSumBCBC)\n .&&(framelessSumBCBCB ?= sparkSumBCBCB)\n }\n\n check(forAll(prop[String, Long, Double, Long, Double] _))\n }\n\n test(\"cube('a, 'b).agg(sum('c), sum('d))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n D: TypedEncoder,\n OutC: TypedEncoder : Numeric,\n OutD: TypedEncoder : Numeric\n ](data: List[X4[A, B, C, D]])(\n implicit\n summableC: CatalystSummable[C, OutC],\n summableD: CatalystSummable[D, OutD]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n\n val framelessSumByAB = dataset\n .cube(A, B)\n .agg(sum(C), sum(D))\n .collect().run().toVector.sortBy(x => (x._1, x._2))\n\n val sparkSumByAB = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\", \"d\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutD](3)))\n .sortBy(x => (x._1, x._2))\n\n framelessSumByAB ?= sparkSumByAB\n }\n\n check(forAll(prop[Byte, Int, Long, Double, Long, Double] _))\n }\n\n test(\"cube('a, 'b).agg(sum('c)) to cube('a, 'b).agg(sum('c),sum('c),sum('c),sum('c),sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n OutC: TypedEncoder: Numeric\n ](data: List[X3[A, B, C]])(implicit summableC: CatalystSummable[C, OutC]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumC = dataset\n .cube(A, B)\n .agg(sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumC = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2)))\n .sortBy(_._2)\n\n val framelessSumCC = dataset\n .cube(A, B)\n .agg(sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCC = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3)))\n .sortBy(_._2)\n\n val framelessSumCCC = dataset\n .cube(A, B)\n .agg(sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCC = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4)))\n .sortBy(_._2)\n\n val framelessSumCCCC = dataset\n .cube(A, B)\n .agg(sum(C), sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCCC = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\", \"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4), row.getAs[OutC](5)))\n .sortBy(_._2)\n\n val framelessSumCCCCC = dataset\n .cube(A, B)\n .agg(sum(C), sum(C), sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCCCC = dataset.dataset\n .cube(\"a\", \"b\").sum(\"c\", \"c\", \"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4), row.getAs[OutC](5), row.getAs[OutC](6)))\n .sortBy(_._2)\n\n (framelessSumC ?= sparkSumC) &&\n (framelessSumCC ?= sparkSumCC) &&\n (framelessSumCCC ?= sparkSumCCC) &&\n (framelessSumCCCC ?= sparkSumCCCC) &&\n (framelessSumCCCCC ?= sparkSumCCCCC)\n }\n\n check(forAll(prop[String, Long, Double, Double] _))\n }\n\n test(\"cube('a, 'b).mapGroups('a, 'b, sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val framelessSumByAB = dataset\n .cube(A, B)\n .deserialized.mapGroups { case ((a, b), xs) => (a, b, xs.map(_.c).sum) }\n .collect().run().toVector.sortBy(x => (x._1, x._2))\n\n val sumByAB = data.groupBy(x => (x.a, x.b))\n .mapValues { xs => xs.map(_.c).sum }\n .toVector.map { case ((a, b), c) => (a, b, c) }.sortBy(x => (x._1, x._2))\n\n framelessSumByAB ?= sumByAB\n }\n\n check(forAll(prop[Byte, Int, Long] _))\n }\n\n test(\"cube('a).mapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder: Ordering,\n B: TypedEncoder: Ordering,\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .cube(A)\n .deserialized.mapGroups((a, xs) => (a, xs.toVector.sorted))\n .collect().run().toMap\n\n val dataGrouped = data.groupBy(_.a).map { case (k, v) => k -> v.sorted }\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"cube('a).flatMapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .cube(A)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run()\n .sorted\n\n val dataGrouped = data\n .groupBy(_.a).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"cube('a, 'b).flatMapGroups((('a,'b) toVector((('a,'b), 'c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Ordering\n ](data: Vector[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val cA = dataset.col[A]('a)\n val cB = dataset.col[B]('b)\n\n val datasetGrouped = dataset\n .cube(cA, cB)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run()\n .sorted\n\n val dataGrouped = data\n .groupBy(t => (t.a, t.b)).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short], Long] _))\n check(forAll(prop[Option[Short], Short, Int] _))\n check(forAll(prop[X1[Option[Short]], Short, Byte] _))\n }\n\n test(\"cubeMany('a).agg(sum('b))\") {\n def prop[A: TypedEncoder : Ordering, Out: TypedEncoder : Numeric]\n (data: List[X1[A]])(implicit summable: CatalystSummable[A, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.cubeMany(A).agg(count[X1[A]]()).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.cube(\"a\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Long](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/ops/PivotTest.scala", "content": "package frameless\npackage ops\n\nimport frameless.functions.aggregate._\nimport org.apache.spark.sql.{functions => sparkFunctions}\nimport org.scalacheck.Arbitrary.arbitrary\nimport org.scalacheck.Prop._\nimport org.scalacheck.{Gen, Prop}\n\nclass PivotTest extends TypedDatasetSuite {\n def withCustomGenX4: Gen[Vector[X4[String, String, Int, Boolean]]] = {\n val kvPairGen: Gen[X4[String, String, Int, Boolean]] = for {\n a <- Gen.oneOf(Seq(\"1\", \"2\", \"3\", \"4\"))\n b <- Gen.oneOf(Seq(\"a\", \"b\", \"c\"))\n c <- arbitrary[Int]\n d <- arbitrary[Boolean]\n } yield X4(a, b, c, d)\n\n Gen.listOfN(4, kvPairGen).map(_.toVector)\n }\n\n test(\"X4[Boolean, String, Int, Boolean] pivot on String\") {\n def prop(data: Vector[X4[String, String, Int, Boolean]]): Prop = {\n val d = TypedDataset.create(data)\n val frameless = d.groupBy(d('a)).\n pivot(d('b)).on(\"a\", \"b\", \"c\").\n agg(sum(d('c)), first(d('d))).collect().run().toVector\n\n val spark = d.dataset.groupBy(\"a\")\n .pivot(\"b\", Seq(\"a\", \"b\", \"c\"))\n .agg(sparkFunctions.sum(\"c\"), sparkFunctions.first(\"d\")).collect().toVector\n\n (frameless.map(_._1) ?= spark.map(x => x.getAs[String](0))).&&(\n frameless.map(_._2) ?= spark.map(x => Option(x.getAs[Long](1)))).&&(\n frameless.map(_._3) ?= spark.map(x => Option(x.getAs[Boolean](2)))).&&(\n frameless.map(_._4) ?= spark.map(x => Option(x.getAs[Long](3)))).&&(\n frameless.map(_._5) ?= spark.map(x => Option(x.getAs[Boolean](4)))).&&(\n frameless.map(_._6) ?= spark.map(x => Option(x.getAs[Long](5)))).&&(\n frameless.map(_._7) ?= spark.map(x => Option(x.getAs[Boolean](6))))\n }\n\n check(forAll(withCustomGenX4)(prop))\n }\n\n test(\"Pivot on Boolean\") {\n val x: Seq[X3[String, Boolean, Boolean]] = Seq(X3(\"a\", true, true), X3(\"a\", true, true), X3(\"a\", true, false))\n val d = TypedDataset.create(x)\n d.groupByMany(d('a)).\n pivot(d('c)).on(true, false).\n agg(count[X3[String, Boolean, Boolean]]()).\n collect().run().toVector ?= Vector((\"a\", Some(2L), Some(1L))) // two true one false\n }\n\n test(\"Pivot with groupBy on two columns, pivot on Long\") {\n val x: Seq[X3[String, String, Long]] = Seq(X3(\"a\", \"x\", 1), X3(\"a\", \"x\", 1), X3(\"a\", \"c\", 20))\n val d = TypedDataset.create(x)\n d.groupBy(d('a), d('b)).\n pivot(d('c)).on(1L, 20L).\n agg(count[X3[String, String, Long]]()).\n collect().run().toSet ?= Set((\"a\", \"x\", Some(2L), None), (\"a\", \"c\", None, Some(1L)))\n }\n\n test(\"Pivot with cube on two columns, pivot on Long\") {\n val x: Seq[X3[String, String, Long]] = Seq(X3(\"a\", \"x\", 1), X3(\"a\", \"x\", 1), X3(\"a\", \"c\", 20))\n val d = TypedDataset.create(x)\n d.cube(d('a), d('b))\n .pivot(d('c)).on(1L, 20L)\n .agg(count[X3[String, String, Long]]())\n .collect().run().toSet ?= Set((\"a\", \"x\", Some(2L), None), (\"a\", \"c\", None, Some(1L)))\n }\n\n test(\"Pivot with cube on Boolean\") {\n val x: Seq[X3[String, Boolean, Boolean]] = Seq(X3(\"a\", true, true), X3(\"a\", true, true), X3(\"a\", true, false))\n val d = TypedDataset.create(x)\n d.cube(d('a)).\n pivot(d('c)).on(true, false).\n agg(count[X3[String, Boolean, Boolean]]()).\n collect().run().toVector ?= Vector((\"a\", Some(2L), Some(1L)))\n }\n\n test(\"Pivot with rollup on two columns, pivot on Long\") {\n val x: Seq[X3[String, String, Long]] = Seq(X3(\"a\", \"x\", 1), X3(\"a\", \"x\", 1), X3(\"a\", \"c\", 20))\n val d = TypedDataset.create(x)\n d.rollup(d('a), d('b))\n .pivot(d('c)).on(1L, 20L)\n .agg(count[X3[String, String, Long]]())\n .collect().run().toSet ?= Set((\"a\", \"x\", Some(2L), None), (\"a\", \"c\", None, Some(1L)))\n }\n\n test(\"Pivot with rollup on Boolean\") {\n val x: Seq[X3[String, Boolean, Boolean]] = Seq(X3(\"a\", true, true), X3(\"a\", true, true), X3(\"a\", true, false))\n val d = TypedDataset.create(x)\n d.rollupMany(d('a)).\n pivot(d('c)).on(true, false).\n agg(count[X3[String, Boolean, Boolean]]()).\n collect().run().toVector ?= Vector((\"a\", Some(2L), Some(1L)))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/ops/RepeatTest.scala", "content": "package frameless\npackage ops\n\nimport shapeless.test.illTyped\nimport shapeless.{::, HNil, Nat}\n\nclass RepeatTest extends TypedDatasetSuite {\n test(\"summoning with implicitly\") {\n implicitly[Repeat.Aux[Int::Boolean::HNil, Nat._1, Int::Boolean::HNil]]\n implicitly[Repeat.Aux[Int::Boolean::HNil, Nat._2, Int::Boolean::Int::Boolean::HNil]]\n implicitly[Repeat.Aux[Int::Boolean::HNil, Nat._3, Int::Boolean::Int::Boolean::Int::Boolean::HNil]]\n implicitly[Repeat.Aux[String::HNil, Nat._5, String::String::String::String::String::HNil]]\n }\n\n test(\"ill typed\") {\n illTyped(\"\"\"implicitly[Repeat.Aux[String::HNil, Nat._5, String::String::String::String::HNil]]\"\"\")\n }\n}" }, { "path": "dataset/src/test/scala/frameless/ops/RollupTests.scala", "content": "package frameless\npackage ops\n\nimport frameless.functions.aggregate._\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass RollupTests extends TypedDatasetSuite {\n\n test(\"rollup('a).agg(count())\") {\n def prop[A: TypedEncoder : Ordering, Out: TypedEncoder : Numeric]\n (data: List[X1[A]])(implicit summable: CatalystSummable[A, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.rollup(A).agg(count()).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.rollup(\"a\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Long](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n\n test(\"rollup('a, 'b).agg(count())\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder, Out: TypedEncoder : Numeric]\n (data: List[X2[A, B]])(implicit summable: CatalystSummable[B, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val received = dataset.rollup(A, B).agg(count()).collect().run().toVector.sortBy(_._3)\n val expected = dataset.dataset.rollup(\"a\", \"b\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[Long](2))).sortBy(_._3)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"rollup('a).agg(sum('b)\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder, Out: TypedEncoder : Numeric]\n (data: List[X2[A, B]])(implicit summable: CatalystSummable[B, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val received = dataset.rollup(A).agg(sum(B)).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.rollup(\"a\").sum(\"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Out](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long, Long] _))\n }\n\n test(\"rollup('a).mapGroups('a, sum('b))\") {\n def prop[A: TypedEncoder : Ordering, B: TypedEncoder : Numeric]\n (data: List[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.rollup(A)\n .deserialized.mapGroups { case (a, xs) => (a, xs.map(_.b).sum) }\n .collect().run().toVector.sortBy(_._1)\n val expected = data.groupBy(_.a).mapValues(_.map(_.b).sum).toVector.sortBy(_._1)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n\n test(\"rollup('a).agg(sum('b), sum('c)) to rollup('a).agg(sum('a), sum('b), sum('a), sum('b), sum('a))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder,\n C: TypedEncoder,\n OutB: TypedEncoder : Numeric,\n OutC: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]])(\n implicit\n summableB: CatalystSummable[B, OutB],\n summableC: CatalystSummable[C, OutC]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumBC = dataset\n .rollup(A)\n .agg(sum(B), sum(C))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBC = dataset.dataset.rollup(\"a\").sum(\"b\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2)))\n .sortBy(_._1)\n\n val framelessSumBCB = dataset\n .rollup(A)\n .agg(sum(B), sum(C), sum(B))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCB = dataset.dataset.rollup(\"a\").sum(\"b\", \"c\", \"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3)))\n .sortBy(_._1)\n\n val framelessSumBCBC = dataset\n .rollup(A)\n .agg(sum(B), sum(C), sum(B), sum(C))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCBC = dataset.dataset.rollup(\"a\").sum(\"b\", \"c\", \"b\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3), row.getAs[OutC](4)))\n .sortBy(_._1)\n\n val framelessSumBCBCB = dataset\n .rollup(A)\n .agg(sum(B), sum(C), sum(B), sum(C), sum(B))\n .collect().run().toVector.sortBy(_._1)\n\n val sparkSumBCBCB = dataset.dataset.rollup(\"a\").sum(\"b\", \"c\", \"b\", \"c\", \"b\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[OutB](1), row.getAs[OutC](2), row.getAs[OutB](3), row.getAs[OutC](4), row.getAs[OutB](5)))\n .sortBy(_._1)\n\n (framelessSumBC ?= sparkSumBC)\n .&&(framelessSumBCB ?= sparkSumBCB)\n .&&(framelessSumBCBC ?= sparkSumBCBC)\n .&&(framelessSumBCBCB ?= sparkSumBCBCB)\n }\n\n check(forAll(prop[String, Long, Double, Long, Double] _))\n }\n\n test(\"rollup('a, 'b).agg(sum('c), sum('d))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n D: TypedEncoder,\n OutC: TypedEncoder : Numeric,\n OutD: TypedEncoder : Numeric\n ](data: List[X4[A, B, C, D]])(\n implicit\n summableC: CatalystSummable[C, OutC],\n summableD: CatalystSummable[D, OutD]\n ): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n val D = dataset.col[D]('d)\n\n val framelessSumByAB = dataset\n .rollup(A, B)\n .agg(sum(C), sum(D))\n .collect().run().toVector.sortBy(_._2)\n\n val sparkSumByAB = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\", \"d\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutD](3)))\n .sortBy(_._2)\n\n framelessSumByAB ?= sparkSumByAB\n }\n\n check(forAll(prop[Byte, Int, Long, Double, Long, Double] _))\n }\n\n test(\"rollup('a, 'b).agg(sum('c)) to rollup('a, 'b).agg(sum('c),sum('c),sum('c),sum('c),sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder,\n OutC: TypedEncoder: Numeric\n ](data: List[X3[A, B, C]])(implicit summableC: CatalystSummable[C, OutC]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n val C = dataset.col[C]('c)\n\n val framelessSumC = dataset\n .rollup(A, B)\n .agg(sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumC = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2)))\n .sortBy(_._2)\n\n val framelessSumCC = dataset\n .rollup(A, B)\n .agg(sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCC = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3)))\n .sortBy(_._2)\n\n val framelessSumCCC = dataset\n .rollup(A, B)\n .agg(sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCC = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4)))\n .sortBy(_._2)\n\n val framelessSumCCCC = dataset\n .rollup(A, B)\n .agg(sum(C), sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCCC = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\", \"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4), row.getAs[OutC](5)))\n .sortBy(_._2)\n\n val framelessSumCCCCC = dataset\n .rollup(A, B)\n .agg(sum(C), sum(C), sum(C), sum(C), sum(C))\n .collect().run().toVector\n .sortBy(_._2)\n\n val sparkSumCCCCC = dataset.dataset\n .rollup(\"a\", \"b\").sum(\"c\", \"c\", \"c\", \"c\", \"c\").collect().toVector\n .map(row => (Option(row.getAs[A](0)), Option(row.getAs[B](1)), row.getAs[OutC](2), row.getAs[OutC](3), row.getAs[OutC](4), row.getAs[OutC](5), row.getAs[OutC](6)))\n .sortBy(_._2)\n\n (framelessSumC ?= sparkSumC) &&\n (framelessSumCC ?= sparkSumCC) &&\n (framelessSumCCC ?= sparkSumCCC) &&\n (framelessSumCCCC ?= sparkSumCCCC) &&\n (framelessSumCCCCC ?= sparkSumCCCCC)\n }\n\n check(forAll(prop[String, Long, Double, Double] _))\n }\n\n test(\"rollup('a, 'b).mapGroups('a, 'b, sum('c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Numeric\n ](data: List[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n val B = dataset.col[B]('b)\n\n val framelessSumByAB = dataset\n .rollup(A, B)\n .deserialized.mapGroups { case ((a, b), xs) => (a, b, xs.map(_.c).sum) }\n .collect().run().toVector.sortBy(x => (x._1, x._2))\n\n val sumByAB = data.groupBy(x => (x.a, x.b))\n .mapValues { xs => xs.map(_.c).sum }\n .toVector.map { case ((a, b), c) => (a, b, c) }.sortBy(x => (x._1, x._2))\n\n framelessSumByAB ?= sumByAB\n }\n\n check(forAll(prop[Byte, Int, Long] _))\n }\n\n test(\"rollup('a).mapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder: Ordering,\n B: TypedEncoder: Ordering\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .rollup(A)\n .deserialized.mapGroups((a, xs) => (a, xs.toVector.sorted))\n .collect().run().toMap\n\n val dataGrouped = data.groupBy(_.a).map { case (k, v) => k -> v.sorted }\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"rollup('a).flatMapGroups(('a, toVector(('a, 'b))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering\n ](data: Vector[X2[A, B]]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val datasetGrouped = dataset\n .rollup(A)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run()\n .sorted\n\n val dataGrouped = data\n .groupBy(_.a).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short]] _))\n check(forAll(prop[Option[Short], Short] _))\n check(forAll(prop[X1[Option[Short]], Short] _))\n }\n\n test(\"rollup('a, 'b).flatMapGroups((('a,'b) toVector((('a,'b), 'c))\") {\n def prop[\n A: TypedEncoder : Ordering,\n B: TypedEncoder : Ordering,\n C: TypedEncoder : Ordering\n ](data: Vector[X3[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n val cA = dataset.col[A]('a)\n val cB = dataset.col[B]('b)\n\n val datasetGrouped = dataset\n .rollup(cA, cB)\n .deserialized.flatMapGroups((a, xs) => xs.map(x => (a, x)))\n .collect().run()\n .sorted\n\n val dataGrouped = data\n .groupBy(t => (t.a, t.b)).toSeq\n .flatMap { case (a, xs) => xs.map(x => (a, x)) }\n .sorted\n\n datasetGrouped ?= dataGrouped\n }\n\n check(forAll(prop[Short, Option[Short], Long] _))\n check(forAll(prop[Option[Short], Short, Int] _))\n check(forAll(prop[X1[Option[Short]], Short, Byte] _))\n }\n\n test(\"rollupMany('a).agg(sum('b))\") {\n def prop[A: TypedEncoder : Ordering, Out: TypedEncoder : Numeric]\n (data: List[X1[A]])(implicit summable: CatalystSummable[A, Out]): Prop = {\n val dataset = TypedDataset.create(data)\n val A = dataset.col[A]('a)\n\n val received = dataset.rollupMany(A).agg(count[X1[A]]()).collect().run().toVector.sortBy(_._2)\n val expected = dataset.dataset.rollup(\"a\").count().collect().toVector\n .map(row => (Option(row.getAs[A](0)), row.getAs[Long](1))).sortBy(_._2)\n\n received ?= expected\n }\n\n check(forAll(prop[Int, Long] _))\n }\n}" }, { "path": "dataset/src/test/scala/frameless/ops/SmartProjectTest.scala", "content": "package frameless\npackage ops\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\n\n\ncase class Foo(i: Int, j: Int, x: String)\ncase class Bar(i: Int, x: String)\ncase class InvalidFooProjectionType(i: Int, x: Boolean)\ncase class InvalidFooProjectionName(i: Int, xerr: String)\n\nclass SmartProjectTest extends TypedDatasetSuite {\n // Lazy needed to prevent initialization anterior to the `beforeAll` hook\n lazy val dataset = TypedDataset.create(Foo(1, 2, \"hi\") :: Foo(2, 3, \"there\") :: Nil)\n\n test(\"project Foo to Bar\") {\n assert(dataset.project[Bar].count().run() === 2)\n }\n\n test(\"project to InvalidFooProjection should not type check\") {\n illTyped(\"dataset.project[InvalidFooProjectionType]\")\n illTyped(\"dataset.project[InvalidFooProjectionName]\")\n }\n\n test(\"X4 to X1,X2,X3,X4 projections\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder, D: TypedEncoder](data: Vector[X4[A, B, C, D]]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.project[X4[A, B, C, D]].collect().run().toVector ?= data\n dataset.project[X3[A, B, C]].collect().run().toVector ?= data.map(x => X3(x.a, x.b, x.c))\n dataset.project[X2[A, B]].collect().run().toVector ?= data.map(x => X2(x.a, x.b))\n dataset.project[X1[A]].collect().run().toVector ?= data.map(x => X1(x.a))\n }\n\n check(forAll(prop[Int, String, X1[String], Boolean] _))\n check(forAll(prop[Short, Long, String, Boolean] _))\n check(forAll(prop[Short, (Boolean, Boolean), String, (Int, Int)] _))\n check(forAll(prop[X2[String, Boolean], (Boolean, Boolean), String, Boolean] _))\n check(forAll(prop[X2[String, Boolean], X3[Boolean, Boolean, Long], String, String] _))\n }\n\n test(\"X3U to X1,X2,X3 projections\") {\n def prop[A: TypedEncoder, B: TypedEncoder, C: TypedEncoder](data: Vector[X3U[A, B, C]]): Prop = {\n val dataset = TypedDataset.create(data)\n\n dataset.project[X3[A, B, C]].collect().run().toVector ?= data.map(x => X3(x.a, x.b, x.c))\n dataset.project[X2[A, B]].collect().run().toVector ?= data.map(x => X2(x.a, x.b))\n dataset.project[X1[A]].collect().run().toVector ?= data.map(x => X1(x.a))\n }\n\n check(forAll(prop[Int, String, X1[String]] _))\n check(forAll(prop[Short, Long, String] _))\n check(forAll(prop[Short, (Boolean, Boolean), String] _))\n check(forAll(prop[X2[String, Boolean], (Boolean, Boolean), String] _))\n check(forAll(prop[X2[String, Boolean], X3[Boolean, Boolean, Long], String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/deserialized/FilterTests.scala", "content": "package frameless\npackage ops\npackage deserialized\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass FilterTests extends TypedDatasetSuite {\n test(\"filter\") {\n def prop[A: TypedEncoder](filterFunction: A => Boolean, data: Vector[A]): Prop =\n TypedDataset.create(data).\n deserialized.\n filter(filterFunction).\n collect().run().toVector =? data.filter(filterFunction)\n\n check(forAll(prop[Int] _))\n check(forAll(prop[String] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/deserialized/FlatMapTests.scala", "content": "package frameless\npackage ops\npackage deserialized\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass FlatMapTests extends TypedDatasetSuite {\n test(\"flatMap\") {\n def prop[A: TypedEncoder, B: TypedEncoder](flatMapFunction: A => Vector[B], data: Vector[A]): Prop =\n TypedDataset.create(data).\n deserialized.\n flatMap(flatMapFunction).\n collect().run().toVector =? data.flatMap(flatMapFunction)\n\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Int, String] _))\n check(forAll(prop[String, Int] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/deserialized/MapPartitionsTests.scala", "content": "package frameless\npackage ops\npackage deserialized\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass MapPartitionsTests extends TypedDatasetSuite {\n test(\"mapPartitions\") {\n def prop[A: TypedEncoder, B: TypedEncoder](mapFunction: A => B, data: Vector[A]): Prop = {\n val lifted: Iterator[A] => Iterator[B] = _.map(mapFunction)\n TypedDataset.create(data).\n deserialized.\n mapPartitions(lifted).\n collect().run().toVector =? data.map(mapFunction)\n }\n\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Int, String] _))\n check(forAll(prop[String, Int] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/deserialized/MapTests.scala", "content": "package frameless\npackage ops\npackage deserialized\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass MapTests extends TypedDatasetSuite {\n test(\"map\") {\n def prop[A: TypedEncoder, B: TypedEncoder](mapFunction: A => B, data: Vector[A]): Prop =\n TypedDataset.create(data).\n deserialized.\n map(mapFunction).\n collect().run().toVector =? data.map(mapFunction)\n\n check(forAll(prop[Int, Int] _))\n check(forAll(prop[Int, String] _))\n check(forAll(prop[String, Int] _))\n check(forAll(prop[X1[Int], X1[Int]] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/ops/deserialized/ReduceTests.scala", "content": "package frameless\npackage ops\npackage deserialized\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\n\nclass ReduceTests extends TypedDatasetSuite {\n def prop[A: TypedEncoder](reduceFunction: (A, A) => A)(data: Vector[A]): Prop =\n TypedDataset.create(data).\n deserialized.\n reduceOption(reduceFunction).run() =? data.reduceOption(reduceFunction)\n\n test(\"reduce Int\") {\n check(forAll(prop[Int](_ + _) _))\n check(forAll(prop[Int](_ * _) _))\n }\n\n test(\"reduce String\") {\n def reduce(s1: String, s2: String): String = (s1 ++ s2).sorted\n check(forAll(prop[String](reduce) _))\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/package.scala", "content": "import java.time.format.DateTimeFormatter\nimport java.time.{LocalDateTime => JavaLocalDateTime}\n\nimport org.scalacheck.{Arbitrary, Gen}\n\npackage object frameless {\n /** Fixed decimal point to avoid precision problems specific to Spark */\n implicit val arbBigDecimal: Arbitrary[BigDecimal] = Arbitrary {\n for {\n x <- Gen.chooseNum(-1000, 1000)\n y <- Gen.chooseNum(0, 1000000)\n } yield BigDecimal(s\"$x.$y\")\n }\n\n /** Fixed decimal point to avoid precision problems specific to Spark */\n implicit val arbDouble: Arbitrary[Double] = Arbitrary {\n arbBigDecimal.arbitrary.map(_.toDouble)\n }\n\n implicit val arbSqlDate = Arbitrary {\n Arbitrary.arbitrary[Int].map(SQLDate)\n }\n\n implicit val arbSqlTimestamp = Arbitrary {\n Arbitrary.arbitrary[Long].map(SQLTimestamp)\n }\n\n implicit def arbTuple1[A: Arbitrary] = Arbitrary {\n Arbitrary.arbitrary[A].map(Tuple1(_))\n }\n\n // see issue with scalacheck non serializable Vector: https://github.com/rickynils/scalacheck/issues/315\n implicit def arbVector[A](implicit A: Arbitrary[A]): Arbitrary[Vector[A]] =\n Arbitrary(Gen.listOf(A.arbitrary).map(_.toVector))\n\n def vectorGen[A: Arbitrary]: Gen[Vector[A]] = arbVector[A].arbitrary\n\n implicit val arbUdtEncodedClass: Arbitrary[UdtEncodedClass] = Arbitrary {\n for {\n int <- Arbitrary.arbitrary[Int]\n doubles <- Gen.listOf(arbDouble.arbitrary)\n } yield new UdtEncodedClass(int, doubles.toArray)\n }\n\n val dateTimeFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm\")\n\n implicit val localDateArb: Arbitrary[JavaLocalDateTime] = Arbitrary {\n for {\n year <- Gen.chooseNum(1900, 2027)\n month <- Gen.chooseNum(1, 12)\n dayOfMonth <- Gen.chooseNum(1, 28)\n hour <- Gen.chooseNum(1, 23)\n minute <- Gen.chooseNum(1, 59)\n } yield JavaLocalDateTime.of(year, month, dayOfMonth, hour, minute)\n }\n\n /** LocalDateTime String Generator to test time related Spark functions */\n val dateTimeStringGen: Gen[List[String]] =\n for {\n listOfDates <- Gen.listOf(localDateArb.arbitrary)\n localDate <- listOfDates\n } yield localDate.format(dateTimeFormatter)\n\n val TEST_OUTPUT_DIR = \"target/test-output\"\n\n /**\n * Will dive down causes until either the cause is true or there are no more causes\n * @param t\n * @param f\n * @return\n */\n def anyCauseHas(t: Throwable, f: Throwable => Boolean): Boolean =\n if (f(t))\n true\n else\n if (t.getCause ne null)\n anyCauseHas(t.getCause, f)\n else\n false\n\n /**\n * Runs up to maxRuns and outputs the number of failures (times thrown)\n * @param maxRuns\n * @param thunk\n * @tparam T\n * @return the last passing thunk, or null\n */\n def runLoads[T](maxRuns: Int = 1000)(thunk: => T): T ={\n var i = 0\n var r = null.asInstanceOf[T]\n var passed = 0\n while(i < maxRuns){\n i += 1\n try {\n r = thunk\n passed += 1\n if (i % 20 == 0) {\n println(s\"run $i successful\")\n }\n } catch {\n case t: Throwable => System.err.println(s\"failed unexpectedly on run $i - ${t.getMessage}\")\n }\n }\n if (passed != maxRuns) {\n System.err.println(s\"had ${maxRuns - passed} failures out of $maxRuns runs\")\n }\n r\n }\n\n /**\n * Runs a given thunk up to maxRuns times, restarting the thunk if tolerantOf the thrown Throwable is true\n * @param tolerantOf\n * @param maxRuns default of 20\n * @param thunk\n * @return either a successful run result or the last error will be thrown\n */\n def tolerantRun[T](tolerantOf: Throwable => Boolean, maxRuns: Int = 20)(thunk: => T): T ={\n var passed = false\n var i = 0\n var res: T = null.asInstanceOf[T]\n var thrown: Throwable = null\n\n while((i < maxRuns) && !passed) {\n try {\n i += 1\n res = thunk\n passed = true\n } catch {\n case t: Throwable if anyCauseHas(t, tolerantOf) =>\n // rinse and repeat\n thrown = t\n case t: Throwable =>\n throw t\n }\n }\n if (!passed) {\n System.err.println(s\"Despite being tolerant each of the $maxRuns runs failed, re-throwing the last\")\n throw thrown\n }\n res\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/sql/package.scala", "content": "package frameless\n\nimport org.apache.spark.sql.catalyst.expressions.Expression\nimport org.apache.spark.sql.catalyst.expressions.{And, Or}\n\npackage object sql {\n implicit class ExpressionOps(val self: Expression) extends AnyVal {\n def toList: List[Expression] = {\n def rec(expr: Expression, acc: List[Expression]): List[Expression] = {\n expr match {\n case And(left, right) => rec(left, rec(right, acc))\n case Or(left, right) => rec(left, rec(right, acc))\n case e => e +: acc\n }\n }\n\n rec(self, Nil)\n }\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/sql/rules/SQLRulesSuite.scala", "content": "package frameless.sql.rules\n\nimport frameless._\nimport frameless.sql._\nimport org.apache.spark.sql.catalyst.expressions.Expression\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.catalyst.plans.logical\nimport org.apache.spark.sql.execution.FileSourceScanExec\nimport org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec\nimport org.scalatest.Assertion\nimport org.scalatest.matchers.should.Matchers\n\ntrait SQLRulesSuite extends TypedDatasetSuite with Matchers { self =>\n protected lazy val path: String = {\n val tmpDir = System.getProperty(\"java.io.tmpdir\")\n s\"$tmpDir/${self.getClass.getName}\"\n }\n\n def withDataset[A: TypedEncoder: CatalystOrdered](payload: A)(f: TypedDataset[A] => Assertion): Assertion = {\n TypedDataset.create(Seq(payload)).write.mode(\"overwrite\").parquet(path)\n f(TypedDataset.createUnsafe[A](session.read.parquet(path)))\n }\n\n def predicatePushDownTest[A: TypedEncoder: CatalystOrdered](\n expected: X1[A],\n expectedPushDownFilters: List[Filter],\n planShouldNotContain: PartialFunction[Expression, Expression],\n op: TypedColumn[X1[A], A] => TypedColumn[X1[A], Boolean]\n ): Assertion = {\n withDataset(expected) { dataset =>\n val ds = dataset.filter(op(dataset('a)))\n val actualPushDownFilters = pushDownFilters(ds)\n\n val optimizedPlan = ds.queryExecution.optimizedPlan.collect { case logical.Filter(condition, _) => condition }.flatMap(_.toList)\n\n // check the optimized plan\n optimizedPlan.collectFirst(planShouldNotContain) should be (empty)\n\n // compare filters\n actualPushDownFilters shouldBe expectedPushDownFilters\n\n val actual = ds.collect().run().toVector.headOption\n\n // ensure serialization is not broken\n actual should be(Some(expected))\n }\n }\n\n protected def pushDownFilters[T](ds: TypedDataset[T]): List[Filter] = {\n val sparkPlan = ds.queryExecution.executedPlan\n\n val initialPlan =\n if (sparkPlan.children.isEmpty) // assume it's AQE\n sparkPlan match {\n case aq: AdaptiveSparkPlanExec => aq.initialPlan\n case _ => sparkPlan\n }\n else\n sparkPlan\n\n initialPlan.collect {\n case fs: FileSourceScanExec =>\n import scala.reflect.runtime.{universe => ru}\n\n val runtimeMirror = ru.runtimeMirror(getClass.getClassLoader)\n val instanceMirror = runtimeMirror.reflect(fs)\n val getter = ru.typeOf[FileSourceScanExec].member(ru.TermName(\"pushedDownFilters\")).asTerm.getter\n val m = instanceMirror.reflectMethod(getter.asMethod)\n val res = m.apply(fs).asInstanceOf[Seq[Filter]]\n\n res\n }.flatten.toList\n }\n}\n" }, { "path": "dataset/src/test/scala/frameless/syntax/FramelessSyntaxTests.scala", "content": "package frameless\npackage syntax\n\nimport org.scalacheck.Prop\nimport org.scalacheck.Prop._\nimport frameless.functions.aggregate._\n\nclass FramelessSyntaxTests extends TypedDatasetSuite {\n // Hide the implicit SparkDelay[Job] on TypedDatasetSuite to avoid ambiguous implicits\n override val sparkDelay = null\n\n def prop[A, B](data: Vector[X2[A, B]])(\n implicit ev: TypedEncoder[X2[A, B]]\n ): Prop = {\n val dataset = TypedDataset.create(data).dataset\n val dataframe = dataset.toDF()\n\n val typedDataset = dataset.typed\n val typedDatasetFromDataFrame = dataframe.unsafeTyped[X2[A, B]]\n\n typedDataset.collect().run().toVector ?= typedDatasetFromDataFrame.collect().run().toVector\n }\n\n test(\"dataset typed - toTyped\") {\n def prop[A, B](data: Vector[X2[A, B]])(\n implicit ev: TypedEncoder[X2[A, B]]\n ): Prop = {\n val dataset = session.createDataset(data)(TypedExpressionEncoder(ev)).typed\n val dataframe = dataset.toDF()\n\n dataset.collect().run().toVector ?= dataframe.unsafeTyped[X2[A, B]].collect().run().toVector\n }\n\n check(forAll(prop[Int, String] _))\n check(forAll(prop[X1[Long], String] _))\n }\n\n test(\"frameless typed column and aggregate\") {\n def prop[A: TypedEncoder](a: A, b: A): Prop = {\n val d = TypedDataset.create((a, b) :: Nil)\n (d.select(d('_1).untyped.typedColumn).collect().run ?= d.select(d('_1)).collect().run).&&(\n d.agg(first(d('_1))).collect().run() ?= d.agg(first(d('_1)).untyped.typedAggregate).collect().run()\n )\n }\n\n check(forAll(prop[Int] _))\n check(forAll(prop[X1[Long]] _))\n }\n}\n" }, { "path": "dataset/src/test/scala/org/apache/hadoop/fs/local/StreamingFS.scala", "content": "package org.apache.hadoop.fs.local\n\nimport com.globalmentor.apache.hadoop.fs.BareLocalFileSystem\nimport org.apache.hadoop.fs.DelegateToFileSystem\n\nclass StreamingFS(uri: java.net.URI, conf: org.apache.hadoop.conf.Configuration) extends\n DelegateToFileSystem(uri, new BareLocalFileSystem(), conf, \"file\", false) {}\n" }, { "path": "dataset/src/test/spark-3.2/frameless/sql/rules/FramelessLitPushDownTests.scala", "content": "package frameless.sql.rules\n\nimport frameless._\nimport frameless.sql._\nimport frameless.functions.Lit\nimport org.apache.spark.sql.catalyst.util.DateTimeUtils.{currentTimestamp, microsToInstant}\nimport org.apache.spark.sql.sources.{Filter, IsNotNull}\nimport org.apache.spark.sql.catalyst.expressions\nimport org.apache.spark.sql.catalyst.expressions.{Cast, Expression, GenericRowWithSchema}\nimport java.time.Instant\n\nimport org.apache.spark.sql.catalyst.plans.logical\nimport org.scalatest.Assertion\n\n//Note as InvokeLike and \"ConditionalExpression\" don't have SPARK-40380 and SPARK-39106 no predicate pushdowns can happen in 3.2.4\nclass FramelessLitPushDownTests extends SQLRulesSuite {\n private val now: Long = currentTimestamp()\n\n test(\"java.sql.Timestamp push-down\") {\n val expected = java.sql.Timestamp.from(microsToInstant(now))\n val expectedStructure = X1(SQLTimestamp(now))\n val expectedPushDownFilters = List(IsNotNull(\"a\"))\n\n predicatePushDownTest[SQLTimestamp](\n expectedStructure,\n expectedPushDownFilters,\n { case e @ expressions.GreaterThanOrEqual(_, _: Lit[_]) => e },\n _ >= expectedStructure.a\n )\n }\n\n test(\"java.time.Instant push-down\") {\n val expected = java.sql.Timestamp.from(microsToInstant(now))\n val expectedStructure = X1(microsToInstant(now))\n val expectedPushDownFilters = List(IsNotNull(\"a\"))\n\n predicatePushDownTest[Instant](\n expectedStructure,\n expectedPushDownFilters,\n { case e @ expressions.GreaterThanOrEqual(_, _: Lit[_]) => e },\n _ >= expectedStructure.a\n )\n }\n\n test(\"struct push-down\") {\n type Payload = X4[Int, Int, Int, Int]\n val expectedStructure = X1(X4(1, 2, 3, 4))\n val expected = new GenericRowWithSchema(Array(1, 2, 3, 4), TypedExpressionEncoder[Payload].schema)\n val expectedPushDownFilters = List(IsNotNull(\"a\"))\n\n predicatePushDownTest[Payload](\n expectedStructure,\n expectedPushDownFilters,\n // Cast not Lit because of SPARK-40380\n { case e @ expressions.EqualTo(_, _: Cast) => e },\n _ === expectedStructure.a\n )\n }\n\n override def predicatePushDownTest[A: TypedEncoder: CatalystOrdered](\n expected: X1[A],\n expectedPushDownFilters: List[Filter],\n planShouldContain: PartialFunction[Expression, Expression],\n op: TypedColumn[X1[A], A] => TypedColumn[X1[A], Boolean]\n ): Assertion = {\n withDataset(expected) { dataset =>\n val ds = dataset.filter(op(dataset('a)))\n val actualPushDownFilters = pushDownFilters(ds)\n\n val optimizedPlan = ds.queryExecution.optimizedPlan.collect { case logical.Filter(condition, _) => condition }.flatMap(_.toList)\n\n // check the optimized plan\n optimizedPlan.collectFirst(planShouldContain) should not be (empty)\n\n // compare filters\n actualPushDownFilters shouldBe expectedPushDownFilters\n\n val actual = ds.collect().run().toVector.headOption\n\n // ensure serialization is not broken\n actual should be(Some(expected))\n }\n }\n\n}\n" }, { "path": "dataset/src/test/spark-3.3+/frameless/sql/rules/FramelessLitPushDownTests.scala", "content": "package frameless.sql.rules\n\nimport frameless._\nimport frameless.functions.Lit\nimport org.apache.spark.sql.catalyst.util.DateTimeUtils.{currentTimestamp, microsToInstant}\nimport org.apache.spark.sql.sources.{EqualTo, GreaterThanOrEqual, IsNotNull}\nimport org.apache.spark.sql.catalyst.expressions\nimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema\nimport java.time.Instant\n\nclass FramelessLitPushDownTests extends SQLRulesSuite {\n private val now: Long = currentTimestamp()\n\n test(\"java.sql.Timestamp push-down\") {\n val expected = java.sql.Timestamp.from(microsToInstant(now))\n val expectedStructure = X1(SQLTimestamp(now))\n val expectedPushDownFilters = List(IsNotNull(\"a\"), GreaterThanOrEqual(\"a\", expected))\n\n predicatePushDownTest[SQLTimestamp](\n expectedStructure,\n expectedPushDownFilters,\n { case e @ expressions.GreaterThanOrEqual(_, _: Lit[_]) => e },\n _ >= expectedStructure.a\n )\n }\n\n test(\"java.time.Instant push-down\") {\n val expected = java.sql.Timestamp.from(microsToInstant(now))\n val expectedStructure = X1(microsToInstant(now))\n val expectedPushDownFilters = List(IsNotNull(\"a\"), GreaterThanOrEqual(\"a\", expected))\n\n predicatePushDownTest[Instant](\n expectedStructure,\n expectedPushDownFilters,\n { case e @ expressions.GreaterThanOrEqual(_, _: Lit[_]) => e },\n _ >= expectedStructure.a\n )\n }\n\n test(\"struct push-down\") {\n type Payload = X4[Int, Int, Int, Int]\n val expectedStructure = X1(X4(1, 2, 3, 4))\n val expected = new GenericRowWithSchema(Array(1, 2, 3, 4), TypedExpressionEncoder[Payload].schema)\n val expectedPushDownFilters = List(IsNotNull(\"a\"), EqualTo(\"a\", expected))\n\n predicatePushDownTest[Payload](\n expectedStructure,\n expectedPushDownFilters,\n { case e @ expressions.EqualTo(_, _: Lit[_]) => e },\n _ === expectedStructure.a\n )\n }\n}\n" }, { "path": "docs/Cats.md", "content": "# Using Cats with Frameless\n\n```scala mdoc:invisible\nimport org.apache.spark.{SparkConf, SparkContext => SC}\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.rdd.RDD\n\nval conf: SparkConf = new SparkConf().setMaster(\"local[4]\").setAppName(\"cats.bec test\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nval sc: SC = spark.sparkContext\n\nspark.sparkContext.setLogLevel(\"WARN\")\nSystem.clearProperty(\"spark.master.port\")\nSystem.clearProperty(\"spark.driver.port\")\nSystem.clearProperty(\"spark.hostPort\")\nSystem.setProperty(\"spark.cleaner.ttl\", \"300\")\n\nimport spark.implicits._\n\nimport cats.syntax.all._\nimport cats.effect.{IO, Sync}\nimport cats.data.ReaderT\n```\n\nThere are two main parts to the `cats` integration offered by Frameless:\n- effect suspension in `TypedDataset` using `cats-effect` and `cats-mtl`\n- `RDD` enhancements using algebraic typeclasses in `cats-kernel`\n\nAll the examples below assume you have previously imported `cats.implicits` and `frameless.cats.implicits`.\n\n*Note that you should not import `frameless.syntax._` together with `frameless.cats.implicits._`.*\n\n```scala mdoc\nimport cats.syntax.all._\nimport frameless.cats.implicits._\n```\n\n## Effect Suspension in typed datasets\n\nAs noted in the section about `Job`, all operations on `TypedDataset` are lazy. The results of\noperations that would normally block on plain Spark APIs are wrapped in a type constructor `F[_]`,\nfor which there exists an instance of `SparkDelay[F]`. This typeclass represents the operation of\ndelaying a computation and capturing an implicit `SparkSession`.\n\nIn the `cats` module, we utilize the typeclasses from `cats-effect` for abstracting over these\neffect types - namely, we provide an implicit `SparkDelay` instance for all `F[_]` for which exists\nan instance of `cats.effect.Sync[F]`.\n\nThis allows one to run operations on `TypedDataset` in an existing monad stack. For example, given\nthis pre-existing monad stack:\n```scala mdoc\nimport frameless.TypedDataset\nimport cats.data.ReaderT\nimport cats.effect.IO\nimport cats.effect.implicits._\n\ntype Action[T] = ReaderT[IO, SparkSession, T]\n```\n\nWe will be able to request that values from `TypedDataset` will be suspended in this stack:\n```scala mdoc\nval typedDs = TypedDataset.create(Seq((1, \"string\"), (2, \"another\")))\nval result: Action[(Seq[(Int, String)], Long)] = for {\n sample <- typedDs.take[Action](1)\n count <- typedDs.count[Action]()\n} yield (sample, count)\n```\n\nAs with `Job`, note that nothing has been run yet. The effect has been properly suspended. To\nrun our program, we must first supply the `SparkSession` to the `ReaderT` layer and then\nrun the `IO` effect:\n```scala mdoc\nimport cats.effect.unsafe.implicits.global\n\nresult.run(spark).unsafeRunSync()\n```\n\n### Convenience methods for modifying Spark thread-local variables\n\nThe `frameless.cats.implicits._` import also provides some syntax enrichments for any monad\nstack that has the same capabilities as `Action` above. Namely, the ability to provide an\ninstance of `SparkSession` and the ability to suspend effects.\n\nFor these to work, we will need to import the implicit machinery from the `cats-mtl` library:\n```scala mdoc\nimport cats.mtl.implicits._\n```\n\nAnd now, we can set the description for the computation being run:\n```scala mdoc\nval resultWithDescription: Action[(Seq[(Int, String)], Long)] = for {\n r <- result.withDescription(\"fancy cats\")\n session <- ReaderT.ask[IO, SparkSession]\n _ <- ReaderT.liftF {\n IO {\n println(s\"Description: ${session.sparkContext.getLocalProperty(\"spark.job.description\")}\")\n }\n }\n} yield r\n\nresultWithDescription.run(spark).unsafeRunSync()\n```\n\n## Using algebraic typeclasses from Cats with RDDs\n\nData aggregation is one of the most important operations when working with Spark (and data in general).\nFor example, we often have to compute the `min`, `max`, `avg`, etc. from a set of columns grouped by\ndifferent predicates. This section shows how **cats** simplifies these tasks in Spark by\nleveraging a large collection of Type Classes for ordering and aggregating data.\n\n\nCats offers ways to sort and aggregate tuples of arbitrary arity.\n\n```scala mdoc\nimport frameless.cats.implicits._\n\nval data: RDD[(Int, Int, Int)] = sc.makeRDD((1, 2, 3) :: (1, 5, 3) :: (8, 2, 3) :: Nil)\n\nprintln(data.csum)\nprintln(data.cmax)\nprintln(data.cmin)\n```\n\nIn case the RDD is empty, the `csum`, `cmax` and `cmin` will use the default values for the type of\nelements inside the RDD. There are counterpart operations to those that have an `Option` return type\nto deal with the case of an empty RDD:\n\n```scala mdoc:nest\nval data: RDD[(Int, Int, Int)] = sc.emptyRDD\n\nprintln(data.csum)\nprintln(data.csumOption)\nprintln(data.cmax)\nprintln(data.cmaxOption)\nprintln(data.cmin)\nprintln(data.cminOption)\n```\n\nThe following example aggregates all the elements with a common key.\n\n```scala mdoc\ntype User = String\ntype TransactionCount = Int\n\nval allData: RDD[(User,TransactionCount)] =\n sc.makeRDD((\"Bob\", 12) :: (\"Joe\", 1) :: (\"Anna\", 100) :: (\"Bob\", 20) :: (\"Joe\", 2) :: Nil)\n\nval totalPerUser = allData.csumByKey\n\ntotalPerUser.collectAsMap\n```\n\nThe same example would work for more complex keys.\n\n```scala mdoc\nimport scala.collection.immutable.SortedMap\n\nval allDataComplexKeu =\n sc.makeRDD( (\"Bob\", SortedMap(\"task1\" -> 10)) ::\n (\"Joe\", SortedMap(\"task1\" -> 1, \"task2\" -> 3)) :: (\"Bob\", SortedMap(\"task1\" -> 10, \"task2\" -> 1)) :: (\"Joe\", SortedMap(\"task3\" -> 4)) :: Nil )\n\nval overalTasksPerUser = allDataComplexKeu.csumByKey\n\noveralTasksPerUser.collectAsMap\n```\n\n#### Joins\n\n```scala mdoc\n// Type aliases for meaningful types\ntype TimeSeries = Map[Int,Int]\ntype UserName = String\n```\n\nExample: Using the implicit full-outer-join operator\n\n```scala mdoc\nimport frameless.cats.outer._\n\nval day1: RDD[(UserName,TimeSeries)] = sc.makeRDD( (\"John\", Map(0 -> 2, 1 -> 4)) :: (\"Chris\", Map(0 -> 1, 1 -> 2)) :: (\"Sam\", Map(0 -> 1)) :: Nil )\nval day2: RDD[(UserName,TimeSeries)] = sc.makeRDD( (\"John\", Map(0 -> 10, 1 -> 11)) :: (\"Chris\", Map(0 -> 1, 1 -> 2)) :: (\"Joe\", Map(0 -> 1, 1 -> 2)) :: Nil )\n\nval daysCombined = day1 |+| day2\n\ndaysCombined.collect()\n```\n\nNote how the user's timeseries from different days have been aggregated together.\nThe `|+|` (Semigroup) operator for key-value pair RDD will execute a full-outer-join\non the key and combine values using the default Semigroup for the value type.\n\nIn `cats`:\n\n```scala mdoc\nMap(1 -> 2, 2 -> 3) |+| Map(1 -> 4, 2 -> -1)\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n" }, { "path": "docs/FeatureOverview.md", "content": "# TypedDataset: Feature Overview\n\nThis tutorial introduces `TypedDataset` using a simple example.\nThe following imports are needed to make all code examples compile.\n\n```scala mdoc:silent:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\nimport frameless.functions.aggregate._\nimport frameless.TypedDataset\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"Frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n```\n\n## Creating TypedDataset instances\n\nWe start by defining a case class:\n\n```scala mdoc:silent\ncase class Apartment(city: String, surface: Int, price: Double, bedrooms: Int)\n```\n\nAnd few `Apartment` instances:\n\n```scala mdoc:silent\nval apartments = Seq(\n Apartment(\"Paris\", 50, 300000.0, 2),\n Apartment(\"Paris\", 100, 450000.0, 3),\n Apartment(\"Paris\", 25, 250000.0, 1),\n Apartment(\"Lyon\", 83, 200000.0, 2),\n Apartment(\"Lyon\", 45, 133000.0, 1),\n Apartment(\"Nice\", 74, 325000.0, 3)\n)\n```\n\nWe are now ready to instantiate a `TypedDataset[Apartment]`:\n\n```scala mdoc\nval aptTypedDs = TypedDataset.create(apartments)\n```\n\nWe can also create one from an existing Spark `Dataset`:\n\n```scala mdoc:nest\nval aptDs = spark.createDataset(apartments)\nval aptTypedDs = TypedDataset.create(aptDs)\n```\n\nOr use the Frameless syntax:\n\n```scala mdoc\nimport frameless.syntax._\n\nval aptTypedDs2 = aptDs.typed\n```\n\n## Typesafe column referencing\n\nThis is how we select a particular column from a `TypedDataset`:\n\n```scala mdoc\nval cities: TypedDataset[String] = aptTypedDs.select(aptTypedDs('city))\n```\n\nThis is completely type-safe, for instance suppose we misspell `city` as `citi`:\n\n```scala mdoc:fail\naptTypedDs.select(aptTypedDs('citi))\n```\n\nThis gets raised at compile time, whereas with the standard `Dataset` API the error appears at runtime (enjoy the stack trace):\n\n```scala mdoc:crash\naptDs.select('citi)\n```\n\n`select()` supports arbitrary column operations:\n\n```scala mdoc\naptTypedDs.select(aptTypedDs('surface) * 10, aptTypedDs('surface) + 2).show().run()\n```\n\nNote that unlike the standard Spark API, where some operations are lazy and some are not, **all TypedDatasets operations are lazy.**\nIn the above example, `show()` is lazy. It requires to apply `run()` for the `show` job to materialize.\nA more detailed explanation of `Job` is given [here](Job.md).\n\nNext we compute the price by surface unit:\n\n```scala mdoc:fail\nval priceBySurfaceUnit = aptTypedDs.select(aptTypedDs('price) / aptTypedDs('surface))\n```\n\nAs the error suggests, we can't divide a `TypedColumn` of `Double` by `Int.`\nFor safety, in Frameless only math operations between same types is allowed:\n\n```scala mdoc\nval priceBySurfaceUnit = aptTypedDs.select(aptTypedDs('price) / aptTypedDs('surface).cast[Double])\npriceBySurfaceUnit.collect().run()\n```\n\nLooks like it worked, but that `cast` seems unsafe right? Actually it is safe.\nLet's try to cast a `TypedColumn` of `String` to `Double`:\n\n```scala mdoc:fail\naptTypedDs('city).cast[Double]\n```\n\nThe compile-time error tells us that to perform the cast, an evidence\n(in the form of `CatalystCast[String, Double]`) must be available.\nSince casting from `String` to `Double` is not allowed, this results\nin a compilation error.\n\nCheck [here](https://github.com/typelevel/frameless/blob/master/core/src/main/scala/frameless/CatalystCast.scala)\nfor the set of available `CatalystCast.`\n\n## Working with Optional columns\n\nWhen working with real data we have to deal with imperfections, such as missing fields. Columns that may have\nmissing data should be represented using `Options`. For this example, let's assume that the Apartments dataset\nmay have missing values. \n\n```scala mdoc:silent\ncase class ApartmentOpt(city: Option[String], surface: Option[Int], price: Option[Double], bedrooms: Option[Int])\n```\n\n```scala mdoc:silent\nval apartmentsOpt = Seq(\n ApartmentOpt(Some(\"Paris\"), Some(50), Some(300000.0), None),\n ApartmentOpt(None, None, Some(450000.0), Some(3))\n)\n```\n\n```scala mdoc\nval aptTypedDsOpt = TypedDataset.create(apartmentsOpt)\naptTypedDsOpt.show().run()\n```\n\nUnfortunately the syntax used above with `select()` will not work here:\n\n```scala mdoc:fail\naptTypedDsOpt.select(aptTypedDsOpt('surface) * 10, aptTypedDsOpt('surface) + 2).show().run()\n```\n\nThis is because we cannot multiple an `Option` with an `Int`. In Scala, `Option` has a `map()` method to help address\nexactly this (e.g., `Some(10).map(c => c * 2)`). Frameless follows a similar convention. By applying the `opt` method on \nany `Option[X]` column you can then use `map()` to provide a function that works with the unwrapped type `X`. \nThis is best shown in the example bellow:\n\n ```scala mdoc\n aptTypedDsOpt.select(aptTypedDsOpt('surface).opt.map(c => c * 10), aptTypedDsOpt('surface).opt.map(_ + 2)).show().run()\n ```\n\n**Known issue**: `map()` will throw a runtime exception when the applied function includes a `udf()`. If you want to \napply a `udf()` to an optional column, we recommend changing your `udf` to work directly with `Optional` fields. \n\n\n## Casting and projections\n\nIn the general case, `select()` returns a TypedDataset of type `TypedDataset[TupleN[...]]` (with N in `[1...10]`).\nFor example, if we select three columns with types `String`, `Int`, and `Boolean` the result will have type\n`TypedDataset[(String, Int, Boolean)]`. \n\nWe often want to give more expressive types to the result of our computations.\n`as[T]` allows us to safely cast a `TypedDataset[U]` to another of type `TypedDataset[T]` as long\nas the types in `U` and `T` align.\n\nWhen the cast is valid the expression compiles:\n\n```scala mdoc\ncase class UpdatedSurface(city: String, surface: Int)\nval updated = aptTypedDs.select(aptTypedDs('city), aptTypedDs('surface) + 2).as[UpdatedSurface]\nupdated.show(2).run()\n```\n\nNext we try to cast a `(String, String)` to an `UpdatedSurface` (which has types `String`, `Int`).\nThe cast is not valid and the expression does not compile:\n\n```scala mdoc:fail\naptTypedDs.select(aptTypedDs('city), aptTypedDs('city)).as[UpdatedSurface]\n```\n\n### Advanced topics with `select()`\n\nWhen you `select()` a single column that has type `A`, the resulting type is `TypedDataset[A]` and \nnot `TypedDataset[Tuple1[A]]`. This behavior makes working with nested schema easier (i.e., in the case \nwhere `A` is a complex data type) and simplifies type-checking column operations (e.g., verify that two \ncolumns can be added, divided, etc.). However, when `A` is scalar, say a `Long`, it makes it harder to select \nand work with the resulting `TypedDataset[Long]`. For instance, it's harder to reference this single scalar \ncolumn using `select()`. If this becomes an issue, you can bypass this behavior by using the \n`selectMany()` method instead of `select()`. In the previous example, `selectMany()` will return\n`TypedDataset[Tuple1[Long]]` and you can reference its single column using the name `_1`. \n`selectMany()` should also be used when you need to select more than 10 columns. \n`select()` has better IDE support and compiles faster than the macro based `selectMany()`, \nso prefer `select()` for the most common use cases.\n\nWhen you are handed a single scalar column TypedDataset (e.g., `TypedDataset[Double]`) \nthe best way to reference its single column is using the `asCol` (short for \"as a column\") method. \nThis is best shown in the example below. We will see more usages of `asCol` later in this tutorial. \n\n```scala mdoc:nest\nval priceBySurfaceUnit = aptTypedDs.select(aptTypedDs('price) / aptTypedDs('surface).cast[Double])\npriceBySurfaceUnit.select(priceBySurfaceUnit.asCol * 2).show(2).run()\n```\n\n\n### Projections\n\nWe often want to work with a subset of the fields in a dataset.\nProjections allow us to easily select our fields of interest\nwhile preserving their initial names and types for extra safety.\n\nHere is an example using the `TypedDataset[Apartment]` with an additional column:\n\n```scala mdoc\nval aptds = aptTypedDs // For shorter expressions\n\ncase class ApartmentDetails(city: String, price: Double, surface: Int, ratio: Double)\nval aptWithRatio =\n aptds.select(\n aptds('city),\n aptds('price),\n aptds('surface),\n aptds('price) / aptds('surface).cast[Double]\n ).as[ApartmentDetails]\n```\n\nSuppose we only want to work with `city` and `ratio`:\n\n```scala mdoc\ncase class CityInfo(city: String, ratio: Double)\n\nval cityRatio = aptWithRatio.project[CityInfo]\n\ncityRatio.show(2).run()\n```\n\nSuppose we only want to work with `price` and `ratio`:\n\n```scala mdoc\ncase class PriceInfo(ratio: Double, price: Double)\n\nval priceInfo = aptWithRatio.project[PriceInfo]\n\npriceInfo.show(2).run()\n```\n\nWe see that the order of the fields does not matter as long as the\nnames and the corresponding types agree. However, if we make a mistake in\nany of the names and/or their types, then we get a compilation error.\n\nSay we make a typo in a field name:\n\n```scala mdoc:silent\ncase class PriceInfo2(ratio: Double, pricEE: Double)\n```\n\n```scala mdoc:fail\naptWithRatio.project[PriceInfo2]\n```\n\nSay we make a mistake in the corresponding type:\n\n```scala mdoc:silent\ncase class PriceInfo3(ratio: Int, price: Double) // ratio should be Double\n```\n\n```scala mdoc:fail\naptWithRatio.project[PriceInfo3]\n```\n\n### Union of TypedDatasets \n\nLets create a projection of our original dataset with a subset of the fields.\n\n```scala mdoc:nest:silent\ncase class ApartmentShortInfo(city: String, price: Double, bedrooms: Int)\n\nval aptTypedDs2: TypedDataset[ApartmentShortInfo] = aptTypedDs.project[ApartmentShortInfo]\n```\n\nThe union of `aptTypedDs2` with `aptTypedDs` uses all the fields of the caller (`aptTypedDs2`)\nand expects the other dataset (`aptTypedDs`) to include all those fields. \nIf field names/types do not match you get a compilation error. \n\n```scala mdoc\naptTypedDs2.union(aptTypedDs).show().run\n```\n\nThe other way around will not compile, since `aptTypedDs2` has only a subset of the fields. \n\n```scala mdoc:fail\naptTypedDs.union(aptTypedDs2).show().run\n```\n\nFinally, as with `project`, `union` will align fields that have same names/types,\nso fields do not have to be in the same order. \n\n## TypedDataset functions and transformations\n\nFrameless supports many of Spark's functions and transformations. \nHowever, whenever a Spark function does not exist in Frameless, \ncalling `.dataset` will expose the underlying \n`Dataset` (from org.apache.spark.sql, the original Spark APIs), \nwhere you can use anything that would be missing from the Frameless' API.\n\nThese are the main imports for Frameless' aggregate and non-aggregate functions.\n\n```scala\nimport frameless.functions._ // For literals\nimport frameless.functions.nonAggregate._ // e.g., concat, abs\nimport frameless.functions.aggregate._ // e.g., count, sum, avg \n```\n\n### Drop/Replace/Add fields\n\n`dropTupled()` drops a single column and results in a tuple-based schema.\n\n```scala mdoc\naptTypedDs2.dropTupled('price): TypedDataset[(String,Int)]\n```\n\nTo drop a column and specify a new schema use `drop()`.\n\n```scala mdoc\ncase class CityBeds(city: String, bedrooms: Int)\nval cityBeds: TypedDataset[CityBeds] = aptTypedDs2.drop[CityBeds] \n```\n\nOften, you want to replace an existing column with a new value.\n \n```scala mdoc\nval inflation = aptTypedDs2.withColumnReplaced('price, aptTypedDs2('price) * 2)\n \ninflation.show(2).run()\n```\n\nOr use a literal instead.\n\n```scala mdoc\nimport frameless.functions.lit\naptTypedDs2.withColumnReplaced('price, lit(0.001)) \n```\n\nAdding a column using `withColumnTupled()` results in a tupled-based schema.\n\n```scala mdoc\naptTypedDs2.withColumnTupled(lit(Array(\"a\",\"b\",\"c\"))).show(2).run()\n```\n\nSimilarly, `withColumn()` adds a column and explicitly expects a schema for the result.\n\n```scala mdoc\ncase class CityBedsOther(city: String, bedrooms: Int, other: List[String])\n\ncityBeds.\n withColumn[CityBedsOther](lit(List(\"a\",\"b\",\"c\"))).\n show(1).run()\n```\n\nTo conditionally change a column use the `when/otherwise` operation. \n\n```scala mdoc\nimport frameless.functions.nonAggregate.when\naptTypedDs2.withColumnTupled(\n when(aptTypedDs2('city) === \"Paris\", aptTypedDs2('price)).\n when(aptTypedDs2('city) === \"Lyon\", lit(1.1)).\n otherwise(lit(0.0))).show(8).run()\n```\n\nA simple way to add a column without losing important schema information is\nto project the entire source schema into a single column using the `asCol()` method.\n\n```scala mdoc\nval c = cityBeds.select(cityBeds.asCol, lit(List(\"a\",\"b\",\"c\")))\nc.show(1).run()\n```\n\nWhen working with Spark's `DataFrames`, you often select all columns using `.select($\"*\", ...)`. \nIn a way, `asCol()` is a typed equivalent of `$\"*\"`. \n\nTo access nested columns, use the `colMany()` method. \n\n```scala mdoc\nc.select(c.colMany('_1, 'city), c('_2)).show(2).run()\n```\n\n### Working with collections\n\n```scala mdoc\nimport frameless.functions._\nimport frameless.functions.nonAggregate._\n```\n\n```scala mdoc\nval t = cityRatio.select(cityRatio('city), lit(List(\"abc\",\"c\",\"d\")))\nt.withColumnTupled(\n arrayContains(t('_2), \"abc\")\n).show(1).run()\n```\n\nIf accidentally you apply a collection function on a column that is not a collection,\nyou get a compilation error.\n\n```scala mdoc:fail\nt.withColumnTupled(\n arrayContains(t('_1), \"abc\")\n)\n```\n\nFlattening columns in Spark is done with the `explode()` method. Unlike vanilla Spark, \nin Frameless `explode()` is part of `TypedDataset` and not a function of a column. \nThis provides additional safety since more than one `explode()` applied in a single \nstatement results in runtime error in vanilla Spark. \n\n```scala mdoc\nval t2 = cityRatio.select(cityRatio('city), lit(List(1,2,3,4)))\nval flattened = t2.explode('_2): TypedDataset[(String, Int)]\nflattened.show(4).run()\n```\n\nHere is an example of how `explode()` may fail in vanilla Spark. The Frameless \nimplementation does not suffer from this problem since, by design, it can only be applied\nto a single column at a time. \n\n```scala mdoc:fail\n{\n import org.apache.spark.sql.functions.{explode => sparkExplode}\n t2.dataset.toDF().select(sparkExplode($\"_2\"), sparkExplode($\"_2\"))\n}\n```\n\n### Collecting data to the driver\n\nIn Frameless all Spark actions (such as `collect()`) are safe.\n\nTake the first element from a dataset (if the dataset is empty return `None`).\n\n```scala mdoc\ncityBeds.headOption.run()\n```\n\nTake the first `n` elements.\n\n```scala mdoc\ncityBeds.take(2).run()\n```\n\n```scala mdoc\ncityBeds.head(3).run()\n```\n\n```scala mdoc\ncityBeds.limit(4).collect().run()\n```\n\n## Sorting columns\n\nOnly column types that can be sorted are allowed to be selected for sorting. \n\n```scala mdoc\naptTypedDs.orderBy(aptTypedDs('city).asc).show(2).run()\n```\n\nThe ordering can be changed by selecting `.acs` or `.desc`. \n\n```scala mdoc\naptTypedDs.orderBy(\n aptTypedDs('city).asc, \n aptTypedDs('price).desc\n).show(2).run()\n```\n\n## User Defined Functions\n\nFrameless supports lifting any Scala function (up to five arguments) to the\ncontext of a particular `TypedDataset`:\n\n```scala mdoc:nest\n// The function we want to use as UDF\nval priceModifier =\n (name: String, price:Double) => if(name == \"Paris\") price * 2.0 else price\n\nval udf = aptTypedDs.makeUDF(priceModifier)\n\nval aptds = aptTypedDs // For shorter expressions\n\nval adjustedPrice = aptds.select(aptds('city), udf(aptds('city), aptds('price)))\n\nadjustedPrice.show().run()\n```\n\n## GroupBy and Aggregations\nLet's suppose we wanted to retrieve the average apartment price in each city\n```scala mdoc\nval priceByCity = aptTypedDs.groupBy(aptTypedDs('city)).agg(avg(aptTypedDs('price)))\npriceByCity.collect().run()\n```\nAgain if we try to aggregate a column that can't be aggregated, we get a compilation error\n```scala mdoc:fail\naptTypedDs.groupBy(aptTypedDs('city)).agg(avg(aptTypedDs('city)))\n```\n\nNext, we combine `select` and `groupBy` to calculate the average price/surface ratio per city:\n\n```scala mdoc:nest\nval aptds = aptTypedDs // For shorter expressions\n\nval cityPriceRatio = aptds.select(aptds('city), aptds('price) / aptds('surface).cast[Double])\n\ncityPriceRatio.groupBy(cityPriceRatio('_1)).agg(avg(cityPriceRatio('_2))).show().run()\n```\n\nWe can also use `pivot` to further group data on a secondary column.\nFor example, we can compare the average price across cities by number of bedrooms.\n\n```scala mdoc\ncase class BedroomStats(\n city: String,\n AvgPriceBeds1: Option[Double], // Pivot values may be missing, so we encode them using Options\n AvgPriceBeds2: Option[Double],\n AvgPriceBeds3: Option[Double],\n AvgPriceBeds4: Option[Double])\n\nval bedroomStats = aptds.\n groupBy(aptds('city)).\n pivot(aptds('bedrooms)).\n on(1,2,3,4). // We only care for up to 4 bedrooms\n agg(avg(aptds('price))).\n as[BedroomStats] // Typesafe casting\n\nbedroomStats.show().run()\n```\n\nWith pivot, collecting data preserves typesafety by\nencoding potentially missing columns with `Option`.\n\n```scala mdoc\nbedroomStats.collect().run().foreach(println)\n```\n\n#### Working with Optional fields\n\nOptional fields can be converted to non-optional using `getOrElse()`. \n\n```scala mdoc\nval sampleStats = bedroomStats.select(\n bedroomStats('AvgPriceBeds2).getOrElse(0.0),\n bedroomStats('AvgPriceBeds3).getOrElse(0.0))\n\nsampleStats.show().run() \n``` \n\nIn addition, optional columns can be flatten using the `.flattenOption` method on `TypedDatset`.\nThe result contains the rows for which the flattened column is not None (or null). The schema\nis automatically adapted to reflect this change.\n\n```scala mdoc\nval flattenStats = bedroomStats.flattenOption('AvgPriceBeds2)\n\n\n// The second Option[Double] is now of type Double, since all 'null' values are removed\nflattenStats: TypedDataset[(String, Option[Double], Double, Option[Double], Option[Double])]\n```\n\nIn a DataFrame, if you just ignore types, this would equivelantly be written as:\n\n```scala mdoc\nbedroomStats.dataset.toDF().filter($\"AvgPriceBeds2\".isNotNull)\n```\n\n### Entire TypedDataset Aggregation\n\nWe often want to aggregate the entire `TypedDataset` and skip the `groupBy()` clause.\nIn Frameless you can do this using the `agg()` operator directly on the `TypedDataset`.\nIn the following example, we compute the average price, the average surface,\nthe minimum surface, and the set of cities for the entire dataset.\n\n```scala mdoc\ncase class Stats(\n avgPrice: Double,\n avgSurface: Double,\n minSurface: Int,\n allCities: Vector[String])\n\naptds.agg(\n avg(aptds('price)),\n avg(aptds('surface)),\n min(aptds('surface)),\n collectSet(aptds('city))\n).as[Stats].show().run()\n```\n\nYou may apply any `TypedColumn` operation to a `TypedAggregate` column as well.\n\n```scala mdoc\nimport frameless.functions._\naptds.agg(\n avg(aptds('price)) * min(aptds('surface)).cast[Double], \n avg(aptds('surface)) * 0.2,\n litAggr(\"Hello World\")\n).show().run()\n```\n\n## Joins\n\n```scala mdoc:silent\ncase class CityPopulationInfo(name: String, population: Int)\n\nval cityInfo = Seq(\n CityPopulationInfo(\"Paris\", 2229621),\n CityPopulationInfo(\"Lyon\", 500715),\n CityPopulationInfo(\"Nice\", 343629)\n)\n\nval citiInfoTypedDS = TypedDataset.create(cityInfo)\n```\n\nHere is how to join the population information to the apartment's dataset:\n\n```scala mdoc\nval withCityInfo = aptTypedDs.joinInner(citiInfoTypedDS) { aptTypedDs('city) === citiInfoTypedDS('name) }\n\nwithCityInfo.show().run()\n```\n\nThe joined TypedDataset has type `TypedDataset[(Apartment, CityPopulationInfo)]`.\n\nWe can then select which information we want to continue to work with:\n\n```scala mdoc\ncase class AptPriceCity(city: String, aptPrice: Double, cityPopulation: Int)\n\nwithCityInfo.select(\n withCityInfo.colMany('_2, 'name), withCityInfo.colMany('_1, 'price), withCityInfo.colMany('_2, 'population)\n).as[AptPriceCity].show().run\n```\n\n### Chained Joins\n\nJoins, or any similar operation, may be chained using a thrush combinator removing the need for intermediate values. Instead of:\n\n```scala mdoc\nval withBedroomInfoInterim = aptTypedDs.joinInner(citiInfoTypedDS)( aptTypedDs('city) === citiInfoTypedDS('name) )\nval withBedroomInfo = withBedroomInfoInterim \n .joinLeft(bedroomStats)( withBedroomInfoInterim.col('_1).field('city) === bedroomStats('city) )\n\nwithBedroomInfo.show().run()\n```\n\nYou can use thrush from [mouse](https://github.com/typelevel/mouse):\n\n```scala\nlibraryDependencies += \"org.typelevel\" %% \"mouse\" % \"1.2.1\"\n```\n\n```scala mdoc\nimport mouse.all._\n\nval withBedroomInfoChained = aptTypedDs.joinInner(citiInfoTypedDS)( aptTypedDs('city) === citiInfoTypedDS('name) )\n .thrush( interim => interim.joinLeft(bedroomStats)( interim.col('_1).field('city) === bedroomStats('city) ) )\n\nwithBedroomInfoChained.show().run()\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n" }, { "path": "docs/Injection.md", "content": "# Injection: Creating Custom Encoders\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\nimport frameless.functions.aggregate._\nimport frameless.TypedDataset\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n```\nInjection lets us define encoders for types that do not have one by injecting `A` into an encodable type `B`.\nThis is the definition of the injection typeclass:\n```scala\ntrait Injection[A, B] extends Serializable {\n def apply(a: A): B\n def invert(b: B): A\n}\n```\n\n## Example\n\nLet's define a simple case class:\n\n```scala mdoc\ncase class Person(age: Int, birthday: java.util.Calendar)\nval people = Seq(Person(42, new java.util.GregorianCalendar()))\n```\n\nAnd an instance of a `TypedDataset`:\n\n```scala mdoc:fail:nest\nval personDS = TypedDataset.create(people)\n```\n\nLooks like we can't, a `TypedEncoder` instance of `Person` is not available, or more precisely for `java.util.Calendar`.\nBut we can define a injection from `java.util.Calendar` to an encodable type, like `Long`:\n\n```scala mdoc\nimport java.util.Calendar\n\nimport frameless._\n\nimplicit val calendarToLongInjection = new Injection[Calendar, Long] {\n def apply(d: Calendar): Long = d.getTime.getTime\n\n def invert(l: Long): Calendar = {\n val cal = new java.util.GregorianCalendar()\n cal.setTime(new java.util.Date(l))\n cal\n }\n}\n```\n\nWe can be less verbose using the `Injection.apply` function:\n\n```scala mdoc:nest\nimport frameless._\n\nimport java.util.Calendar\n\nimplicit val calendarToLongInjection = Injection[Calendar, Long](\n (_: Calendar).getTime.getTime,\n { (l: Long) =>\n val cal = new java.util.GregorianCalendar()\n cal.setTime(new java.util.Date(l))\n cal\n })\n```\n\nNow we can create our `TypedDataset`:\n\n```scala mdoc\nval personDS = TypedDataset.create(people)\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n\n## Another example\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\nimport frameless.functions.aggregate._\nimport frameless.TypedDataset\n\nval conf = new SparkConf().\n setMaster(\"local[*]\").\n setAppName(\"frameless repl\").\n set(\"spark.ui.enabled\", \"false\")\n\nimplicit val spark = SparkSession.builder().\n config(conf).appName(\"REPL\").getOrCreate()\n\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n```\n\nLet's define a sealed family:\n\n```scala mdoc\nsealed trait Gender\ncase object Male extends Gender\ncase object Female extends Gender\ncase object Other extends Gender\n```\n\nAnd a simple case class:\n\n```scala mdoc\ncase class Person(age: Int, gender: Gender)\nval people = Seq(Person(42, Male))\n```\n\nAgain if we try to create a `TypedDataset`, we get a compilation error.\n\n```scala mdoc:fail:nest\nval personDS = TypedDataset.create(people)\n```\n\nLet's define an injection instance for `Gender`:\n\n```scala mdoc\nimport frameless._\n\nimplicit val genderToInt: Injection[Gender, Int] = Injection(\n {\n case Male => 1\n case Female => 2\n case Other => 3\n },\n {\n case 1 => Male\n case 2 => Female\n case 3 => Other\n })\n```\n\nAnd now we can create our `TypedDataset`:\n\n```scala mdoc\nval personDS = TypedDataset.create(people)\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n\nAlternatively, an injection instance can be derived for sealed families such as `Gender` using the following \nimport, `import frameless.TypedEncoder.injections._`. This will encode the data constructors as strings.\n\n**Known issue**: An invalid injection instance will be derived if there are data constructors with the same name.\nFor example, consider the following sealed family:\n\n```scala mdoc\nsealed trait Foo\nobject A { case object Bar extends Foo }\nobject B { case object Bar extends Foo }\n```\n\n`A.Bar` and `B.Bar` will both be encoded as `\"Bar\"` thereby breaking the law that `invert(apply(x)) == x`.\n" }, { "path": "docs/Job.md", "content": "# Job\\[A\\]\n\nAll operations on `TypedDataset` are lazy. An operation either returns a new\ntransformed `TypedDataset` or an `F[A]`, where `F[_]` is a type constructor\nwith an instance of the `SparkDelay` typeclass and `A` is the result of running a\nnon-lazy computation in Spark. \n\nA default such type constructor called `Job` is provided by Frameless. \n\n`Job` serves several functions:\n- Makes all operations on a `TypedDataset` lazy, which makes them more predictable compared to having\nfew operations being lazy and other being strict\n- Allows the programmer to make expensive blocking operations explicit\n- Allows for Spark jobs to be lazily sequenced using monadic composition via for-comprehension\n- Provides an obvious place where you can annotate/name your Spark jobs to make it easier\nto track different parts of your application in the Spark UI\n\nThe toy example showcases the use of for-comprehension to explicitly sequences Spark Jobs.\nFirst we calculate the size of the `TypedDataset` and then we collect to the driver\nexactly 20% of its elements:\n\n```scala mdoc:invisible\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\nimport frameless.functions.aggregate._\nimport frameless.TypedDataset\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n```\n\n```scala mdoc\nimport frameless.syntax._\n\nval ds = TypedDataset.create(1 to 20)\n\nval countAndTakeJob =\n for {\n count <- ds.count()\n sample <- ds.take((count/5).toInt)\n } yield sample\n\ncountAndTakeJob.run()\n```\n\nThe `countAndTakeJob` can either be executed using `run()` (as we show above) or it can\nbe passed along to other parts of the program to be further composed into more complex sequences\nof Spark jobs.\n\n```scala mdoc\nimport frameless.Job\ndef computeMinOfSample(sample: Job[Seq[Int]]): Job[Int] = sample.map(_.min)\n\nval finalJob = computeMinOfSample(countAndTakeJob)\n```\n\nNow we can execute this new job by specifying a [group-id][group-id] and a description.\nThis allows the programmer to see this information on the Spark UI and help track, say,\nperformance issues.\n\n```scala mdoc\nfinalJob.\n withGroupId(\"samplingJob\").\n withDescription(\"Samples 20% of elements and computes the min\").\n run()\n```\n\n\n```scala mdoc:invisible\nspark.stop()\n```\n\n[group-id]: https://spark.apache.org/docs/latest/api/scala/index.html#org.apache.spark.SparkContext@setJobGroup(groupId:String,description:String,interruptOnCancel:Boolean):Unit\n\n## More on `SparkDelay`\n\nAs mentioned above, `SparkDelay[F[_]]` is a typeclass required for suspending\neffects by Spark computations. This typeclass represents the ability to suspend\nan `=> A` thunk into an `F[A]` value, while implicitly capturing a `SparkSession`.\n\nAs it is a typeclass, it is open for implementation by the user in order to use\nother data types for suspension of effects. The `cats` module, for example, uses\nthis typeclass to support suspending Spark computations in any effect type that\nhas a `cats.effect.Sync` instance.\n" }, { "path": "docs/TypedDataFrame.md", "content": "# Proof of Concept: TypedDataFrame\n\n`TypedDataFrame` is the API developed in the early stages of Frameless to manipulate Spark `DataFrame`s in a type-safe manner. With the introduction of `Dataset` in Spark 1.6, `DataFrame` seems deprecated and won't be the focus of future development of Frameless. However, the design is interesting enough to document.\n\nTo safely manipulate `DataFrame`s we use a technique called a *shadow type*, which consists in storing additional information about a value in a \"dummy\" type. Mirroring value-level computation at the type-level lets us leverage the type system to catch common mistakes at compile time.\n\n### Diving in\n\nIn `TypedDataFrame`, we use a single `Schema <: Product` to model the number, the types and the names of columns. Here is a what the definition of `TypedDataFrame` looks like, with simplified type signatures:\n\n```scala\nimport org.apache.spark.sql.DataFrame\nimport shapeless.HList\n\nclass TDataFrame[Schema <: Product](df: DataFrame) {\n def filter(predicate: Schema => Boolean): TDataFrame[Schema] = ???\n\n def select[C <: HList, Out <: Product](columns: C): TDataFrame[Out] = ???\n\n def innerJoin[OtherS <: Product, Out <: Product]\n (other: TDataFrame[OtherS]): TDataFrame[Out] = ???\n\n // Followed by equivalent of every DataFrame method with improved signature\n}\n```\n\nAs you can see, instead of the `def filter(conditionExpr: String): DataFrame` defined in Spark, the `TypedDataFrame` version expects a function from `Schema` to `Boolean`, and models the fact that resulting `DataFrame` will still hold elements of type `Schema`.\n\n### Type-level column referencing\n\nFor Spark's `DataFrame`s, column referencing is done directly by `String`s or using the `Column` type which provides no additional type safety. `TypedDataFrame` improves on that by catching invalid column references compile type. When everything goes well, Frameless select is very similar to vanilla select, except that it keeps track of the selected column types:\n\n```scala\nimport frameless.TypedDataFrame\n\ncase class Foo(s: String, d: Double, i: Int)\n\ndef selectIntString(tf: TypedDataFrame[Foo]): TypedDataFrame[(Int, String)] =\n tf.select('i, 's)\n```\n\nHowever, in case of typo, it gets caught right away:\n\n```scala\ndef selectIntStringTypo(tf: TypedDataFrame[Foo]): TypedDataFrame[(Int, String)] =\n tf.select('j, 's)\n```\n\n### Type-level joins\n\nJoins can available with two different syntaxes. The first lets you reference different columns on each `TypedDataFrame`, and ensures that they all exist and have compatible types:\n\n```scala\ncase class Bar(i: Int, j: String, b: Boolean)\n\ndef join1(tf1: TypedDataFrame[Foo], tf2: TypedDataFrame[Bar])\n : TypedDataFrame[(String, Double, Int, Int, String, Boolean)] =\n tf1.innerJoin(tf2).on('s).and('j)\n```\n\nThe second syntax brings some convenience when the joining columns have identical names in both tables:\n\n```scala\ndef join2(tf1: TypedDataFrame[Foo], tf2: TypedDataFrame[Bar])\n : TypedDataFrame[(String, Double, Int, String, Boolean)] =\n tf1.innerJoin(tf2).using('i)\n```\n\nFurther example are available in the [TypedDataFrame join tests.](https://github.com/typelevel/frameless/blob/17194d2172e75f8994e9481181e85b4c8dcc0f69/dataframe/src/test/scala/JoinTests.scala)\n\n### Complete example\n\nWe now consider a complete example to see how the Frameless types can improve not only correctness but also the readability of Spark jobs. Consider the following domain of phonebooks, city maps and neighborhoods:\n\n```scala mdoc:silent\ntype Neighborhood = String\ntype Address = String\n\ncase class PhoneBookEntry(\n address: Address,\n residents: String,\n phoneNumber: Double\n)\n\ncase class CityMapEntry(\n address: Address,\n neighborhood: Neighborhood\n)\n```\n\nOur goal will be to compute the neighborhood with unique names, approximating \"unique\" with names containing less common\nletters in the alphabet: 'x', 'q', and 'z'. We are going to need a natural language processing library at some point, so\nlet's use the following for the example:\n\n```scala mdoc:silent\nobject NLPLib {\n def uniqueName(name: String): Boolean = name.exists(Set('x', 'q', 'z'))\n}\n```\n\nSuppose we manage to obtain public data for a `TypedDataFrame[PhoneBookEntry]` and `TypedDataFrame[CityMapEntry]`. Here is what our Spark job could look like with Frameless:\n\n```scala\nimport org.apache.spark.sql.SQLContext\n\n// These case classes are used to hold intermediate results\ncase class Family(residents: String, neighborhood: Neighborhood)\ncase class Person(name: String, neighborhood: Neighborhood)\ncase class NeighborhoodCount(neighborhood: Neighborhood, count: Long)\n\ndef bestNeighborhood\n (phoneBookTF: TypedDataFrame[PhoneBookEntry], cityMapTF: TypedDataFrame[CityMapEntry])\n (implicit c: SQLContext): String = {\n (((((((((\n phoneBookTF\n .innerJoin(cityMapTF).using('address) :TypedDataFrame[(Address, String, Double, String)])\n .select('_2, '_4) :TypedDataFrame[(String, String)])\n .as[Family]() :TypedDataFrame[Family])\n .flatMap { f =>\n f.residents.split(' ').map(r => Person(r, f.neighborhood))\n } :TypedDataFrame[Person])\n .filter { p =>\n NLPLib.uniqueName(p.name)\n } :TypedDataFrame[Person])\n .groupBy('neighborhood).count() :TypedDataFrame[(String, Long)])\n .as[NeighborhoodCount]() :TypedDataFrame[NeighborhoodCount])\n .sortDesc('count) :TypedDataFrame[NeighborhoodCount])\n .select('neighborhood) :TypedDataFrame[Tuple1[String]])\n .head._1\n}\n```\n\nIf you compare this version to vanilla Spark where every line is a `DataFrame`, you see how much types can improve readability. An executable version of this example is available in the [BestNeighborhood test](https://github.com/typelevel/frameless/blob/17194d2172e75f8994e9481181e85b4c8dcc0f69/dataframe/src/test/scala/BestNeighborhood.scala).\n\n### Limitations\n\nThe main limitation of this approach comes from Scala 2.10, which limits the arity of class classes to 22. Because of the way `DataFrame` models joins, joining two table with more that 11 fields results in a `DataFrame` which not representable with `Schema` of type `Product`.\n\nIn the `Dataset` API introduced in Spark 1.6, the way join are handled was rethought to return a pair of both schemas instead of a flat table, which moderates the trouble caused by case class limitations. Alternatively, since Scala 2.11, it is possible to define Tuple23 and onward. Sadly, due to the way Spark is commonly packaged in various systems, the amount Spark users having to Scala 2.11 and *not* to Spark 1.6 is essentially zero. For this reasons, further development in Frameless will target Spark 1.6+, deprecating the early work on`TypedDataFrame`.\n" }, { "path": "docs/TypedDatasetVsSparkDataset.md", "content": "# Comparing TypedDatasets with Spark's Datasets\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"test\").set(\"spark.ui.enabled\", \"false\").set(\"spark.app.id\", \"tut-dataset\")\nimplicit val spark = SparkSession.builder().config(conf).getOrCreate()\n\nSystem.clearProperty(\"spark.master.port\")\nSystem.clearProperty(\"spark.driver.port\")\nSystem.clearProperty(\"spark.hostPort\")\nSystem.setProperty(\"spark.cleaner.ttl\", \"300\")\n\n// We are using this directory so let's make sure it is clean first\norg.apache.commons.io.FileUtils.deleteDirectory(new java.io.File(\"/tmp/foo/\"))\n```\n\n**Goal:**\n This tutorial compares the standard Spark Datasets API with the one provided by\n Frameless' `TypedDataset`. It shows how `TypedDataset`s allow for an expressive and\n type-safe api with no compromises on performance.\n\nFor this tutorial we first create a simple dataset and save it on disk as a parquet file.\n[Parquet](https://parquet.apache.org/) is a popular columnar format and well supported by Spark.\nIt's important to note that when operating on parquet datasets, Spark knows that each column is stored\nseparately, so if we only need a subset of the columns Spark will optimize for this and avoid reading\nthe entire dataset. This is a rather simplistic view of how Spark and parquet work together but it\nwill serve us well for the context of this discussion.\n\n```scala mdoc\nimport spark.implicits._\n\n// Our example case class Foo acting here as a schema\ncase class Foo(i: Long, j: String)\n\n// Assuming spark is loaded and SparkSession is bind to spark\nval initialDs = spark.createDataset( Foo(1, \"Q\") :: Foo(10, \"W\") :: Foo(100, \"E\") :: Nil )\n\n// Assuming you are on Linux or Mac OS\ninitialDs.write.parquet(\"/tmp/foo\")\n\nval ds = spark.read.parquet(\"/tmp/foo\").as[Foo]\n\nds.show()\n```\n\nThe value `ds` holds the content of the `initialDs` read from a parquet file.\nLet's try to only use field `i` from Foo and see how Spark's Catalyst (the query optimizer)\noptimizes this.\n\n```scala mdoc\n// Using a standard Spark TypedColumn in select()\nval filteredDs = ds.filter($\"i\" === 10).select($\"i\".as[Long])\n\nfilteredDs.show()\n```\n\nThe `filteredDs` is of type `Dataset[Long]`. Since we only access field `i` from `Foo` the type is correct.\nUnfortunately, this syntax requires handholding by explicitly setting the `TypedColumn` in the `select` statement\nto return type `Long` (look at the `as[Long]` statement). We will discuss this limitation next in more detail.\nNow, let's take a quick look at the optimized Physical Plan that Spark's Catalyst generated.\n\n```scala mdoc\nfilteredDs.explain()\n```\n\nThe last line is very important (see `ReadSchema`). The schema read\nfrom the parquet file only required reading column `i` without needing to access column `j`.\nThis is great! We have both an optimized query plan and type-safety!\n\nUnfortunately, this syntax is not bulletproof: it fails at run-time if we try to access\na non existing column `x`:\n\n\n```scala mdoc:crash\nds.filter($\"i\" === 10).select($\"x\".as[Long])\n```\n\nThere are two things to improve here. First, we would want to avoid the `as[Long]` casting that we are required\nto type for type-safety. This is clearly an area where we may introduce a bug by casting to an incompatible\ntype. Second, we want a solution where reference to a non existing column name fails at compilation time.\nThe standard Spark Dataset can achieve this using the following syntax.\n\n```scala mdoc\nds.filter(_.i == 10).map(_.i).show()\n```\n\nThis looks great! It reminds us the familiar syntax from Scala.\nThe two closures in filter and map are functions that operate on `Foo` and the\ncompiler will helps us capture all the mistakes we mentioned above.\n\n```scala mdoc:fail\nds.filter(_.i == 10).map(_.x).show()\n```\n\nUnfortunately, this syntax does not allow Spark to optimize the code.\n\n```scala mdoc\nds.filter(_.i == 10).map(_.i).explain()\n```\n\nAs we see from the explained Physical Plan, Spark was not able to optimize our query as before.\nReading the parquet file will required loading all the fields of `Foo`. This might be ok for\nsmall datasets or for datasets with few columns, but will be extremely slow for most practical\napplications. Intuitively, Spark currently does not have a way to look inside the code we pass in these two\nclosures. It only knows that they both take one argument of type `Foo`, but it has no way of knowing if\nwe use just one or all of `Foo`'s fields.\n\nThe `TypedDataset` in Frameless solves this problem. It allows for a simple and type-safe syntax\nwith a fully optimized query plan.\n\n```scala mdoc\nimport frameless.TypedDataset\nimport frameless.syntax._\nval fds = TypedDataset.create(ds)\n\nfds.filter(fds('i) === 10).select(fds('i)).show().run()\n```\n\nAnd the optimized Physical Plan:\n\n```scala mdoc\nfds.filter(fds('i) === 10).select(fds('i)).explain()\n```\n\nAnd the compiler is our friend.\n\n```scala mdoc:fail\nfds.filter(fds('i) === 10).select(fds('x))\n```\n\n## Differences in Encoders\n\nEncoders in Spark's `Datasets` are partially type-safe. If you try to create a `Dataset` using a type that is not \n a Scala `Product` then you get a compilation error:\n\n```scala mdoc\nclass Bar(i: Int)\n```\n\n`Bar` is neither a case class nor a `Product`, so the following correctly gives a compilation error in Spark:\n\n```scala mdoc:fail\nspark.createDataset(Seq(new Bar(1)))\n```\n\nHowever, the compile type guards implemented in Spark are not sufficient to detect non encodable members. \nFor example, using the following case class leads to a runtime failure:\n\n```scala mdoc\ncase class MyDate(jday: java.util.Calendar)\n```\n\n```scala mdoc:crash\nspark.createDataset(Seq(MyDate {\n val cal = new java.util.GregorianCalendar()\n cal.setTime(new java.util.Date(System.currentTimeMillis))\n cal\n}))\n```\n\nIn comparison, a `TypedDataset` will notify about the encoding problem at compile time: \n\n```scala mdoc:fail\nTypedDataset.create(Seq(MyDate {\n val cal = new java.util.GregorianCalendar()\n cal.setTime(new java.util.Date(System.currentTimeMillis))\n cal\n}))\n```\n\n## Aggregate vs Projected columns \n\nSpark's `Dataset` do not distinguish between columns created from aggregate operations, \nsuch as summing or averaging, and simple projections/selections. \nThis is problematic when you start mixing the two.\n\n```scala mdoc\nimport org.apache.spark.sql.functions.sum\n```\n\n```scala mdoc:crash\nds.select(sum($\"i\"), $\"i\"*2)\n```\n\nIn Frameless, mixing the two results in a compilation error.\n\n```scala mdoc\n// To avoid confusing frameless' sum with the standard Spark's sum\nimport frameless.functions.aggregate.{sum => fsum}\n```\n\n```scala mdoc:fail\nfds.select(fsum(fds('i)))\n```\n\nAs the error suggests, we expected a `TypedColumn` but we got a `TypedAggregate` instead. \n\nHere is how you apply an aggregation method in Frameless: \n\n```scala mdoc\nfds.agg(fsum(fds('i))+22).show().run()\n```\n\nSimilarly, mixing projections while aggregating does not make sense, and in Frameless\nyou get a compilation error. \n\n```scala mdoc:fail\nfds.agg(fsum(fds('i)), fds('i)).show().run()\n```\n\n\n```scala mdoc:invisible\norg.apache.commons.io.FileUtils.deleteDirectory(new java.io.File(\"/tmp/foo/\"))\nspark.stop()\n```\n" }, { "path": "docs/TypedEncoder.md", "content": "# Typed Encoders in Frameless\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\nimport frameless.functions.aggregate._\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nSystem.setProperty(\"spark.cleaner.ttl\", \"300\")\n```\n\nSpark uses Reflection to derive its `Encoder`s, which is why they can fail at run time. For example, because Spark does not support `java.util.Calendar`, the following leads to an error:\n\n```scala mdoc:silent\nimport java.util.Calendar\n\nimport org.apache.spark.sql.Dataset\n\nimport spark.implicits._\n\ncase class DateRange(s: Calendar, e: Calendar)\n```\n\n```scala mdoc:crash\ndef now = new java.util.GregorianCalendar()\n\nval ds: Dataset[DateRange] = Seq(DateRange(now, now)).toDS()\n```\n\nAs shown by the stack trace, this runtime error goes through [ScalaReflection](https://github.com/apache/spark/blob/19cf208063f035d793d2306295a251a9af7e32f6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala) to try to derive an `Encoder` for `Dataset` schema. Beside the annoyance of not detecting this error at compile time, a more important limitation of the reflection-based approach is its inability to be extended for custom types. See this Stack Overflow question for a summary of the current situation (as of 2.0) in vanilla Spark: [How to store custom objects in a Dataset?](http://stackoverflow.com/a/39442829/2311362).\n\nFrameless introduces a new type class called `TypeEncoder` to solve these issues. `TypeEncoder`s are passed around as implicit parameters to every Frameless method to ensure that the data being manipulated is `Encoder`. It uses a standard implicit resolution coupled with shapeless' type class derivation mechanism to ensure every that compiling code manipulates encodable data. For example, the `java.util.Calendar` example won't compile with Frameless:\n\n```scala mdoc:silent\nimport frameless.TypedDataset\nimport frameless.syntax._\n```\n\n```scala mdoc:fail\ndef now = new java.util.GregorianCalendar()\n\nval ds: TypedDataset[DateRange] = TypedDataset.create(Seq(DateRange(now, now)))\n```\n\nType class derivation takes care of recursively constructing (and proving the existence of) `TypeEncoder`s for case classes. The following works as expected:\n\n```scala mdoc\ncase class Bar(d: Double, s: String)\ncase class Foo(i: Int, b: Bar)\n\nval ds: TypedDataset[Foo] = \n TypedDataset.create(Seq(Foo(1, Bar(1.1, \"s\"))))\n\nds.collect()\n```\n\nBut any non-encodable in the case class hierarchy will be detected at compile time:\n\n```scala mdoc:silent\ncase class BarDate(d: Double, s: String, t: java.util.Calendar)\ncase class FooDate(i: Int, b: BarDate)\n```\n\n```scala mdoc:fail\nval ds: TypedDataset[FooDate] = TypedDataset.create(\n Seq(FooDate(1, BarDate(1.1, \"s\", new java.util.GregorianCalendar))))\n```\n\nIt should be noted that once derived, reflection-based `Encoder`s and implicitly derived `TypeEncoder`s have identical performance.\nThe derivation mechanism is different, but the objects generated to encode and decode JVM objects in Spark's internal representation behave the same at runtime.\n\n```scala mdoc:invisible\nspark.stop()\n```\n" }, { "path": "docs/TypedML.md", "content": "# Typed Spark ML\n\nThe `frameless-ml` module provides a strongly typed Spark ML API leveraging `TypedDataset`s. It introduces `TypedTransformer`s\nand `TypedEstimator`s, the type-safe equivalents of Spark ML's `Transformer` and `Estimator`. \n\nA `TypedEstimator` fits models to data, i.e trains a ML model based on an input `TypedDataset`. \nA `TypedTransformer` transforms one `TypedDataset` into another, usually by appending column(s) to it.\n\nBy calling the `fit` method of a `TypedEstimator`, the `TypedEstimator` will train a ML model using the `TypedDataset` \npassed as input (representing the training data) and will return a `TypedTransformer` that represents the trained model. \nThis `TypedTransformer`can then be used to make predictions on an input `TypedDataset` (representing the test data) \nusing the `transform` method that will return a new `TypedDataset` with appended prediction column(s).\n\nBoth `TypedEstimator` and `TypedTransformer` check at compile-time the correctness of their inputs field names and types,\ncontrary to Spark ML API which only deals with DataFrames (the data structure with the lowest level of type-safety in Spark).\n\n`frameless-ml` adds type-safety to Spark ML API but stays very close to it in terms of abstractions and API calls, so \nplease check [Spark ML documentation](https://spark.apache.org/docs/2.2.0/ml-pipeline.html) for more details \non `Transformer`s and `Estimator`s.\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"Frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n```\n\n## Example 1: predict a continuous value using a `TypedRandomForestRegressor`\n\nIn this example, we want to predict the sale price of a house depending on its square footage and the fact that the house\nhas a garden or not. We will use a `TypedRandomForestRegressor`.\n\n### Training\n\nAs with the Spark ML API, we use a `TypedVectorAssembler` (the type-safe equivalent of `VectorAssembler`)\nto compute feature vectors:\n\n```scala mdoc:silent\nimport frameless._\nimport frameless.syntax._\nimport frameless.ml._\nimport frameless.ml.feature._\nimport frameless.ml.regression._\nimport org.apache.spark.ml.linalg.Vector\n```\n\n```scala mdoc\ncase class HouseData(squareFeet: Double, hasGarden: Boolean, price: Double)\n\nval trainingData = TypedDataset.create(Seq(\n HouseData(20, false, 100000),\n HouseData(50, false, 200000),\n HouseData(50, true, 250000),\n HouseData(100, true, 500000)\n))\n\ncase class Features(squareFeet: Double, hasGarden: Boolean)\nval assembler = TypedVectorAssembler[Features]\n\ncase class HouseDataWithFeatures(squareFeet: Double, hasGarden: Boolean, price: Double, features: Vector)\nval trainingDataWithFeatures = assembler.transform(trainingData).as[HouseDataWithFeatures]\n```\n\nIn the above code snippet, `.as[HouseDataWithFeatures]` is a `TypedDataset`'s type-safe cast\n(see [TypedDataset: Feature Overview](https://typelevel.org/frameless/FeatureOverview.html)):\n\n```scala mdoc:silent\ncase class WrongHouseFeatures(\n squareFeet: Double,\n hasGarden: Int, // hasGarden has wrong type\n price: Double,\n features: Vector\n)\n```\n\n```scala mdoc:fail\nassembler.transform(trainingData).as[WrongHouseFeatures]\n```\n\nMoreover, `TypedVectorAssembler[Features]` will compile only if `Features` contains exclusively fields of type Numeric or Boolean:\n\n```scala mdoc:silent\ncase class WrongFeatures(squareFeet: Double, hasGarden: Boolean, city: String)\n```\n\n```scala mdoc:fail\nTypedVectorAssembler[WrongFeatures]\n```\n\nThe subsequent call `assembler.transform(trainingData)` compiles only if `trainingData` contains all fields (names and types)\nof `Features`:\n\n```scala mdoc\ncase class WrongHouseData(squareFeet: Double, price: Double) // hasGarden is missing\nval wrongTrainingData = TypedDataset.create(Seq(WrongHouseData(20, 100000)))\n```\n\n```scala mdoc:fail\nassembler.transform(wrongTrainingData)\n```\n\nThen, we train the model. To train a Random Forest, one needs to feed it with features (what we predict from) and\nwith a label (what we predict). In our example, `price` is the label, `features` are the features:\n\n```scala mdoc\ncase class RFInputs(price: Double, features: Vector)\nval rf = TypedRandomForestRegressor[RFInputs]\n\nval model = rf.fit(trainingDataWithFeatures).run()\n```\n\n`TypedRandomForestRegressor[RFInputs]` compiles only if `RFInputs`\ncontains only one field of type Double (the label) and one field of type Vector (the features):\n\n```scala mdoc:silent\ncase class WrongRFInputs(labelOfWrongType: String, features: Vector)\n```\n\n```scala mdoc:fail\nTypedRandomForestRegressor[WrongRFInputs]\n```\n\nThe subsequent `rf.fit(trainingDataWithFeatures)` call compiles only if `trainingDataWithFeatures` contains the same fields\n(names and types) as RFInputs.\n\n```scala mdoc\nval wrongTrainingDataWithFeatures = TypedDataset.create(Seq(HouseData(20, false, 100000))) // features are missing\n```\n\n```scala mdoc:fail\nrf.fit(wrongTrainingDataWithFeatures) \n```\n\n### Prediction\n\nWe now want to predict `price` for `testData` using the previously trained model. Like the Spark ML API,\n`testData` has a default value for `price` (`0` in our case) that will be ignored at prediction time. We reuse\nour `assembler` to compute the feature vector of `testData`.\n\n```scala mdoc\nval testData = TypedDataset.create(Seq(HouseData(70, true, 0)))\nval testDataWithFeatures = assembler.transform(testData).as[HouseDataWithFeatures]\n\ncase class HousePricePrediction(\n squareFeet: Double,\n hasGarden: Boolean,\n price: Double,\n features: Vector,\n predictedPrice: Double\n)\nval predictions = model.transform(testDataWithFeatures).as[HousePricePrediction]\n\npredictions.select(predictions.col('predictedPrice)).collect.run()\n```\n\n`model.transform(testDataWithFeatures)` will only compile if `testDataWithFeatures` contains a field `price` of type Double\nand a field `features` of type Vector:\n\n```scala mdoc:fail\nmodel.transform(testData)\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n\n## Example 2: predict a categorical value using a `TypedRandomForestClassifier`\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"Frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\nimport frameless._\nimport frameless.syntax._\nimport frameless.ml._\nimport frameless.ml.feature._\nimport frameless.ml.regression._\nimport org.apache.spark.ml.linalg.Vector\n```\n\nIn this example, we want to predict in which city a house is located depending on its price and its square footage. We use a\n`TypedRandomForestClassifier`.\n\n### Training\n\nAs with the Spark ML API, we use a `TypedVectorAssembler` to compute feature vectors and a `TypedStringIndexer`\nto index `city` values in order to be able to pass them to a `TypedRandomForestClassifier`\n(which only accepts Double values as label):\n\n```scala mdoc:silent\nimport frameless.ml.classification._\n```\n\n```scala mdoc\ncase class HouseData(squareFeet: Double, city: String, price: Double)\n\nval trainingData = TypedDataset.create(Seq(\n HouseData(100, \"lyon\", 100000),\n HouseData(200, \"lyon\", 200000),\n HouseData(100, \"san francisco\", 500000),\n HouseData(150, \"san francisco\", 900000)\n))\n\ncase class Features(price: Double, squareFeet: Double)\nval vectorAssembler = TypedVectorAssembler[Features]\n\ncase class HouseDataWithFeatures(squareFeet: Double, city: String, price: Double, features: Vector)\nval dataWithFeatures = vectorAssembler.transform(trainingData).as[HouseDataWithFeatures]\n\ncase class StringIndexerInput(city: String)\nval indexer = TypedStringIndexer[StringIndexerInput]\nindexer.estimator.setHandleInvalid(\"keep\")\nval indexerModel = indexer.fit(dataWithFeatures).run()\n\ncase class HouseDataWithFeaturesAndIndex(\n squareFeet: Double,\n city: String,\n price: Double,\n features: Vector,\n cityIndexed: Double\n)\nval indexedData = indexerModel.transform(dataWithFeatures).as[HouseDataWithFeaturesAndIndex]\n```\n\nThen, we train the model:\n\n```scala mdoc\ncase class RFInputs(cityIndexed: Double, features: Vector)\nval rf = TypedRandomForestClassifier[RFInputs]\n\nval model = rf.fit(indexedData).run()\n```\n\n### Prediction\n\nWe now want to predict `city` for `testData` using the previously trained model. Like the Spark ML API,\n`testData` has a default value for `city` (empty string in our case) that will be ignored at prediction time. We reuse\nour `vectorAssembler` to compute the feature vector of `testData` and our `indexerModel` to index `city`.\n\n```scala mdoc\nval testData = TypedDataset.create(Seq(HouseData(120, \"\", 800000)))\n\nval testDataWithFeatures = vectorAssembler.transform(testData).as[HouseDataWithFeatures]\nval indexedTestData = indexerModel.transform(testDataWithFeatures).as[HouseDataWithFeaturesAndIndex]\n\ncase class HouseCityPredictionInputs(features: Vector, cityIndexed: Double)\nval testInput = indexedTestData.project[HouseCityPredictionInputs]\n\ncase class HouseCityPredictionIndexed(\n features: Vector,\n cityIndexed: Double,\n rawPrediction: Vector,\n probability: Vector,\n predictedCityIndexed: Double\n)\nval indexedPredictions = model.transform(testInput).as[HouseCityPredictionIndexed]\n```\n\nThen, we use a `TypedIndexToString` to get back a String value from `predictedCityIndexed`. `TypedIndexToString` takes\nas input the label array computed by our previous `indexerModel`:\n\n```scala mdoc\ncase class IndexToStringInput(predictedCityIndexed: Double)\n\nval indexToString = TypedIndexToString[IndexToStringInput](indexerModel.transformer.labels)\n\ncase class HouseCityPrediction(\n features: Vector,\n cityIndexed: Double,\n rawPrediction: Vector,\n probability: Vector,\n predictedCityIndexed: Double,\n predictedCity: String\n)\nval predictions = indexToString.transform(indexedPredictions).as[HouseCityPrediction]\n\npredictions.select(predictions.col('predictedCity)).collect.run()\n```\n\n## List of currently implemented `TypedEstimator`s\n\n* `TypedRandomForestClassifier`\n* `TypedRandomForestRegressor`\n* ... [your contribution here](https://github.com/typelevel/frameless/issues/215) ... :)\n\n## List of currently implemented `TypedTransformer`s\n\n* `TypedIndexToString`\n* `TypedStringIndexer`\n* `TypedVectorAssembler`\n* ... [your contribution here](https://github.com/typelevel/frameless/issues/215) ... :)\n \n## Using Vector and Matrix with `TypedDataset`\n\n`frameless-ml` provides `TypedEncoder` instances for `org.apache.spark.ml.linalg.Vector` \nand `org.apache.spark.ml.linalg.Matrix`:\n\n```scala mdoc:silent\nimport frameless._\nimport frameless.ml._\nimport org.apache.spark.ml.linalg._\n```\n\n```scala mdoc\nval vector = Vectors.dense(1, 2, 3)\nval vectorDs = TypedDataset.create(Seq(\"label\" -> vector))\n\nval matrix = Matrices.dense(2, 1, Array(1, 2))\nval matrixDs = TypedDataset.create(Seq(\"label\" -> matrix))\n```\n\nUnder the hood, Vector and Matrix are encoded using `org.apache.spark.ml.linalg.VectorUDT` \nand `org.apache.spark.ml.linalg.MatrixUDT`. This is possible thanks to the implicit derivation \nfrom `org.apache.spark.sql.types.UserDefinedType[A]` to `TypedEncoder[A]` defined in `TypedEncoder` companion object.\n\n```scala mdoc:invisible\nspark.stop()\n```\n" }, { "path": "docs/WorkingWithCsvParquetJson.md", "content": "# Working with CSV and Parquet data\n\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"Frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n\nval testDataPath: String = \"docs/iris.data\"\n```\nYou need these imports for most Frameless projects. \n\n```scala mdoc:silent\nimport frameless._\nimport frameless.syntax._\nimport frameless.functions.aggregate._\n```\n\n## Working with CSV \n\nWe first load some CSV data and print the schema. \n\n```scala mdoc\nval df = spark.read.format(\"csv\").load(testDataPath)\ndf.show(2)\ndf.printSchema\n```\n\nThe easiest way to read from CSV into a `TypedDataset` is to create a case class that follows \nthe exact number, type, and order for the fields as they appear in the CSV file. This is shown in \nthe example bellow with the use of the `Iris` case class.\n\n```scala mdoc\nfinal case class Iris(sLength: Double, sWidth: Double, pLength: Double, pWidth: Double, kind: String)\nval testDataDf = spark.read.format(\"csv\").schema(TypedExpressionEncoder[Iris].schema).load(testDataPath)\nval data: TypedDataset[Iris] = TypedDataset.createUnsafe[Iris](testDataDf)\ndata.show(2).run()\n```\n\nIf we do not explicitly define the schema of the CSV file then the types will not match leading to runtime errors. \n\n```scala mdoc:nest\nval testDataNoSchema = spark.read.format(\"csv\").load(testDataPath)\nval data: TypedDataset[Iris] = TypedDataset.createUnsafe[Iris](testDataNoSchema)\n```\n\n```scala mdoc:crash\ndata.collect().run()\n```\n\n### Dealing with CSV files with multiple columns\n\nWhen the dataset has many columns, it is impractical to define a case class that contains many columns we don't need. \nIn such case, we can project the columns we do need, cast them to the proper type, and then call `createUnsafe` using a case class\nthat contains a much smaller subset of the columns. \n\n```scala mdoc:nest\nimport org.apache.spark.sql.types.DoubleType\nfinal case class IrisLight(kind: String, sLength: Double)\n\nval testDataDf = spark.read.format(\"csv\").load(testDataPath)\nval projectedDf = testDataDf.select(testDataDf(\"_c4\").as(\"kind\"), testDataDf(\"_c1\").cast(DoubleType).as(\"sLength\"))\nval data = TypedDataset.createUnsafe[IrisLight](projectedDf)\ndata.take(2).run()\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n\n## Working with Parquet\n```scala mdoc:invisible:reset-object\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.SparkSession\n\nval conf = new SparkConf().setMaster(\"local[*]\").setAppName(\"Frameless repl\").set(\"spark.ui.enabled\", \"false\")\nimplicit val spark = SparkSession.builder().config(conf).appName(\"REPL\").getOrCreate()\nspark.sparkContext.setLogLevel(\"WARN\")\n\nimport spark.implicits._\n\nval testDataPathParquet: String = \"docs/iris.parquet\"\nimport frameless._\nimport frameless.syntax._\nimport frameless.functions.aggregate._\n\nfinal case class Iris(sLength: Double, sWidth: Double, pLength: Double, pWidth: Double, kind: String)\n```\n\nSpark is much better at reading the schema from parquet files. \n\n```scala mdoc\nval testDataParquet = spark.read.format(\"parquet\").load(testDataPathParquet)\ntestDataParquet.printSchema\n```\n\nSo as long as we use a type (case class) that reflects the same number, type, and order of the fields \nfrom the data everything works as expected. \n\n```scala mdoc:nest\nval data: TypedDataset[Iris] = TypedDataset.createUnsafe[Iris](testDataParquet)\ndata.take(2).run()\n```\n\n### Dealing with Parquet files with multiple columns\n\nThe main difference compared to CSV is that with Parquet Spark is better at inferring the types. This makes it simpler \nto project the columns we need without having the cast the to the proper type. \n\n```scala mdoc:nest\nfinal case class IrisLight(kind: String, sLength: Double)\n\nval projectedDf = testDataParquet.select(\"kind\", \"sLength\")\nval data = TypedDataset.createUnsafe[IrisLight](projectedDf)\ndata.take(2).run()\n```\n\n```scala mdoc:invisible\nspark.stop()\n```\n" }, { "path": "docs/directory.conf", "content": "laika.title = frameless\nlaika.navigationOrder = [\n README.md\n FeatureOverview.md\n TypedDatasetVsSparkDataset.md\n WorkingWithCsvParquetJson.md\n Injection.md\n Job.md\n Cats.md\n TypedML.md\n TypedDataFrame.md\n]" }, { "path": "docs/iris.data", "content": "5.1,3.5,1.4,0.2,Iris-setosa\n4.9,3.0,1.4,0.2,Iris-setosa\n4.7,3.2,1.3,0.2,Iris-setosa\n4.6,3.1,1.5,0.2,Iris-setosa\n5.0,3.6,1.4,0.2,Iris-setosa\n5.4,3.9,1.7,0.4,Iris-setosa\n4.6,3.4,1.4,0.3,Iris-setosa\n5.0,3.4,1.5,0.2,Iris-setosa\n4.4,2.9,1.4,0.2,Iris-setosa\n4.9,3.1,1.5,0.1,Iris-setosa\n5.4,3.7,1.5,0.2,Iris-setosa\n4.8,3.4,1.6,0.2,Iris-setosa\n4.8,3.0,1.4,0.1,Iris-setosa\n4.3,3.0,1.1,0.1,Iris-setosa\n5.8,4.0,1.2,0.2,Iris-setosa\n5.7,4.4,1.5,0.4,Iris-setosa\n5.4,3.9,1.3,0.4,Iris-setosa\n5.1,3.5,1.4,0.3,Iris-setosa\n5.7,3.8,1.7,0.3,Iris-setosa\n5.1,3.8,1.5,0.3,Iris-setosa\n5.4,3.4,1.7,0.2,Iris-setosa\n5.1,3.7,1.5,0.4,Iris-setosa\n4.6,3.6,1.0,0.2,Iris-setosa\n5.1,3.3,1.7,0.5,Iris-setosa\n4.8,3.4,1.9,0.2,Iris-setosa\n5.0,3.0,1.6,0.2,Iris-setosa\n5.0,3.4,1.6,0.4,Iris-setosa\n5.2,3.5,1.5,0.2,Iris-setosa\n5.2,3.4,1.4,0.2,Iris-setosa\n4.7,3.2,1.6,0.2,Iris-setosa\n4.8,3.1,1.6,0.2,Iris-setosa\n5.4,3.4,1.5,0.4,Iris-setosa\n5.2,4.1,1.5,0.1,Iris-setosa\n5.5,4.2,1.4,0.2,Iris-setosa\n4.9,3.1,1.5,0.1,Iris-setosa\n5.0,3.2,1.2,0.2,Iris-setosa\n5.5,3.5,1.3,0.2,Iris-setosa\n4.9,3.1,1.5,0.1,Iris-setosa\n4.4,3.0,1.3,0.2,Iris-setosa\n5.1,3.4,1.5,0.2,Iris-setosa\n5.0,3.5,1.3,0.3,Iris-setosa\n4.5,2.3,1.3,0.3,Iris-setosa\n4.4,3.2,1.3,0.2,Iris-setosa\n5.0,3.5,1.6,0.6,Iris-setosa\n5.1,3.8,1.9,0.4,Iris-setosa\n4.8,3.0,1.4,0.3,Iris-setosa\n5.1,3.8,1.6,0.2,Iris-setosa\n4.6,3.2,1.4,0.2,Iris-setosa\n5.3,3.7,1.5,0.2,Iris-setosa\n5.0,3.3,1.4,0.2,Iris-setosa\n7.0,3.2,4.7,1.4,Iris-versicolor\n6.4,3.2,4.5,1.5,Iris-versicolor\n6.9,3.1,4.9,1.5,Iris-versicolor\n5.5,2.3,4.0,1.3,Iris-versicolor\n6.5,2.8,4.6,1.5,Iris-versicolor\n5.7,2.8,4.5,1.3,Iris-versicolor\n6.3,3.3,4.7,1.6,Iris-versicolor\n4.9,2.4,3.3,1.0,Iris-versicolor\n6.6,2.9,4.6,1.3,Iris-versicolor\n5.2,2.7,3.9,1.4,Iris-versicolor\n5.0,2.0,3.5,1.0,Iris-versicolor\n5.9,3.0,4.2,1.5,Iris-versicolor\n6.0,2.2,4.0,1.0,Iris-versicolor\n6.1,2.9,4.7,1.4,Iris-versicolor\n5.6,2.9,3.6,1.3,Iris-versicolor\n6.7,3.1,4.4,1.4,Iris-versicolor\n5.6,3.0,4.5,1.5,Iris-versicolor\n5.8,2.7,4.1,1.0,Iris-versicolor\n6.2,2.2,4.5,1.5,Iris-versicolor\n5.6,2.5,3.9,1.1,Iris-versicolor\n5.9,3.2,4.8,1.8,Iris-versicolor\n6.1,2.8,4.0,1.3,Iris-versicolor\n6.3,2.5,4.9,1.5,Iris-versicolor\n6.1,2.8,4.7,1.2,Iris-versicolor\n6.4,2.9,4.3,1.3,Iris-versicolor\n6.6,3.0,4.4,1.4,Iris-versicolor\n6.8,2.8,4.8,1.4,Iris-versicolor\n6.7,3.0,5.0,1.7,Iris-versicolor\n6.0,2.9,4.5,1.5,Iris-versicolor\n5.7,2.6,3.5,1.0,Iris-versicolor\n5.5,2.4,3.8,1.1,Iris-versicolor\n5.5,2.4,3.7,1.0,Iris-versicolor\n5.8,2.7,3.9,1.2,Iris-versicolor\n6.0,2.7,5.1,1.6,Iris-versicolor\n5.4,3.0,4.5,1.5,Iris-versicolor\n6.0,3.4,4.5,1.6,Iris-versicolor\n6.7,3.1,4.7,1.5,Iris-versicolor\n6.3,2.3,4.4,1.3,Iris-versicolor\n5.6,3.0,4.1,1.3,Iris-versicolor\n5.5,2.5,4.0,1.3,Iris-versicolor\n5.5,2.6,4.4,1.2,Iris-versicolor\n6.1,3.0,4.6,1.4,Iris-versicolor\n5.8,2.6,4.0,1.2,Iris-versicolor\n5.0,2.3,3.3,1.0,Iris-versicolor\n5.6,2.7,4.2,1.3,Iris-versicolor\n5.7,3.0,4.2,1.2,Iris-versicolor\n5.7,2.9,4.2,1.3,Iris-versicolor\n6.2,2.9,4.3,1.3,Iris-versicolor\n5.1,2.5,3.0,1.1,Iris-versicolor\n5.7,2.8,4.1,1.3,Iris-versicolor\n6.3,3.3,6.0,2.5,Iris-virginica\n5.8,2.7,5.1,1.9,Iris-virginica\n7.1,3.0,5.9,2.1,Iris-virginica\n6.3,2.9,5.6,1.8,Iris-virginica\n6.5,3.0,5.8,2.2,Iris-virginica\n7.6,3.0,6.6,2.1,Iris-virginica\n4.9,2.5,4.5,1.7,Iris-virginica\n7.3,2.9,6.3,1.8,Iris-virginica\n6.7,2.5,5.8,1.8,Iris-virginica\n7.2,3.6,6.1,2.5,Iris-virginica\n6.5,3.2,5.1,2.0,Iris-virginica\n6.4,2.7,5.3,1.9,Iris-virginica\n6.8,3.0,5.5,2.1,Iris-virginica\n5.7,2.5,5.0,2.0,Iris-virginica\n5.8,2.8,5.1,2.4,Iris-virginica\n6.4,3.2,5.3,2.3,Iris-virginica\n6.5,3.0,5.5,1.8,Iris-virginica\n7.7,3.8,6.7,2.2,Iris-virginica\n7.7,2.6,6.9,2.3,Iris-virginica\n6.0,2.2,5.0,1.5,Iris-virginica\n6.9,3.2,5.7,2.3,Iris-virginica\n5.6,2.8,4.9,2.0,Iris-virginica\n7.7,2.8,6.7,2.0,Iris-virginica\n6.3,2.7,4.9,1.8,Iris-virginica\n6.7,3.3,5.7,2.1,Iris-virginica\n7.2,3.2,6.0,1.8,Iris-virginica\n6.2,2.8,4.8,1.8,Iris-virginica\n6.1,3.0,4.9,1.8,Iris-virginica\n6.4,2.8,5.6,2.1,Iris-virginica\n7.2,3.0,5.8,1.6,Iris-virginica\n7.4,2.8,6.1,1.9,Iris-virginica\n7.9,3.8,6.4,2.0,Iris-virginica\n6.4,2.8,5.6,2.2,Iris-virginica\n6.3,2.8,5.1,1.5,Iris-virginica\n6.1,2.6,5.6,1.4,Iris-virginica\n7.7,3.0,6.1,2.3,Iris-virginica\n6.3,3.4,5.6,2.4,Iris-virginica\n6.4,3.1,5.5,1.8,Iris-virginica\n6.0,3.0,4.8,1.8,Iris-virginica\n6.9,3.1,5.4,2.1,Iris-virginica\n6.7,3.1,5.6,2.4,Iris-virginica\n6.9,3.1,5.1,2.3,Iris-virginica\n5.8,2.7,5.1,1.9,Iris-virginica\n6.8,3.2,5.9,2.3,Iris-virginica\n6.7,3.3,5.7,2.5,Iris-virginica\n6.7,3.0,5.2,2.3,Iris-virginica\n6.3,2.5,5.0,1.9,Iris-virginica\n6.5,3.0,5.2,2.0,Iris-virginica\n6.2,3.4,5.4,2.3,Iris-virginica\n5.9,3.0,5.1,1.8,Iris-virginica\n\n" }, { "path": "github.sbt", "content": "ThisBuild / githubWorkflowArtifactUpload := false // doesn't work with scoverage\n\nThisBuild / githubWorkflowEnv += \"SPARK_LOCAL_IP\" -> \"localhost\"\n\nThisBuild / githubWorkflowArtifactDownloadExtraKeys += \"project\"\n\nThisBuild / githubWorkflowBuildSbtStepPreamble += s\"project $${{ matrix.project }}\"\nThisBuild / tlCiScalafmtCheck := true\nThisBuild / githubWorkflowBuild ~= { steps =>\n steps.map { // replace the test step\n case step: WorkflowStep.Sbt if step.commands == List(\"test\") =>\n WorkflowStep.Sbt(\n commands = List(\"coverage\", \"test\", \"test/coverageReport\"),\n name = Some(\"Test & Compute Coverage\")\n )\n case step => step\n }\n}\n\nThisBuild / githubWorkflowBuildPostamble +=\n WorkflowStep.Use(\n UseRef.Public(\n \"codecov\",\n \"codecov-action\",\n \"v3\"\n ),\n params = Map(\"flags\" -> s\"$${{ matrix.scala }}-$${{ matrix.project }}\")\n )\n" }, { "path": "ml/src/main/scala/frameless/ml/TypedEstimator.scala", "content": "package frameless\npackage ml\n\nimport frameless.ops.SmartProject\nimport org.apache.spark.ml.{Estimator, Model}\n\n/**\n * A TypedEstimator fits models to data.\n */\ntrait TypedEstimator[Inputs, Outputs, M <: Model[M]] {\n val estimator: Estimator[M]\n\n def fit[T, F[_]](ds: TypedDataset[T])(\n implicit\n smartProject: SmartProject[T, Inputs],\n F: SparkDelay[F]\n ): F[AppendTransformer[Inputs, Outputs, M]] = {\n implicit val sparkSession = ds.dataset.sparkSession\n F.delay {\n val inputDs = smartProject.apply(ds)\n val model = estimator.fit(inputDs.dataset)\n new AppendTransformer[Inputs, Outputs, M] {\n val transformer: M = model\n }\n }\n }\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/TypedTransformer.scala", "content": "package frameless\npackage ml\n\nimport frameless.ops.SmartProject\nimport org.apache.spark.ml.Transformer\nimport shapeless.{Generic, HList}\nimport shapeless.ops.hlist.{Prepend, Tupler}\n\n/**\n * A TypedTransformer transforms one TypedDataset into another.\n */\nsealed trait TypedTransformer\n\n/**\n * An AppendTransformer `transform` method takes as input a TypedDataset containing `Inputs` and\n * return a TypedDataset with `Outputs` columns appended to the input TypedDataset.\n */\ntrait AppendTransformer[Inputs, Outputs, InnerTransformer <: Transformer] extends TypedTransformer {\n val transformer: InnerTransformer\n\n def transform[T, TVals <: HList, OutputsVals <: HList, OutVals <: HList, Out](ds: TypedDataset[T])(\n implicit\n i0: SmartProject[T, Inputs],\n i1: Generic.Aux[T, TVals],\n i2: Generic.Aux[Outputs, OutputsVals],\n i3: Prepend.Aux[TVals, OutputsVals, OutVals],\n i4: Tupler.Aux[OutVals, Out],\n i5: TypedEncoder[Out]\n ): TypedDataset[Out] = {\n val transformed = transformer.transform(ds.dataset).as[Out](TypedExpressionEncoder[Out])\n TypedDataset.create[Out](transformed)\n }\n\n}\n\nobject AppendTransformer {\n // Random name to a temp column added by a TypedTransformer (the proper name will be given by the Tuple-based encoder)\n private[ml] val tempColumnName = \"I1X3T9CU1OP0128JYIO76TYZZA3AXHQ18RMI\"\n private[ml] val tempColumnName2 = \"I1X3T9CU1OP0128JYIO76TYZZA3AXHQ18RMJ\"\n private[ml] val tempColumnName3 = \"I1X3T9CU1OP0128JYIO76TYZZA3AXHQ18RMK\"\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/classification/TypedRandomForestClassifier.scala", "content": "package frameless\npackage ml\npackage classification\n\nimport frameless.ml.internals.TreesInputsChecker\nimport frameless.ml.params.trees.FeatureSubsetStrategy\nimport org.apache.spark.ml.classification.{RandomForestClassificationModel, RandomForestClassifier}\nimport org.apache.spark.ml.linalg.Vector\n\n/**\n * Random Forest learning algorithm for\n * classification.\n * It supports both binary and multiclass labels, as well as both continuous and categorical\n * features.\n */\nfinal class TypedRandomForestClassifier[Inputs] private[ml](\n rf: RandomForestClassifier,\n labelCol: String,\n featuresCol: String\n) extends TypedEstimator[Inputs, TypedRandomForestClassifier.Outputs, RandomForestClassificationModel] {\n\n val estimator: RandomForestClassifier =\n rf\n .setLabelCol(labelCol)\n .setFeaturesCol(featuresCol)\n .setPredictionCol(AppendTransformer.tempColumnName)\n .setRawPredictionCol(AppendTransformer.tempColumnName2)\n .setProbabilityCol(AppendTransformer.tempColumnName3)\n\n def setNumTrees(value: Int): TypedRandomForestClassifier[Inputs] = copy(rf.setNumTrees(value))\n def setMaxDepth(value: Int): TypedRandomForestClassifier[Inputs] = copy(rf.setMaxDepth(value))\n def setMinInfoGain(value: Double): TypedRandomForestClassifier[Inputs] = copy(rf.setMinInfoGain(value))\n def setMinInstancesPerNode(value: Int): TypedRandomForestClassifier[Inputs] = copy(rf.setMinInstancesPerNode(value))\n def setMaxMemoryInMB(value: Int): TypedRandomForestClassifier[Inputs] = copy(rf.setMaxMemoryInMB(value))\n def setSubsamplingRate(value: Double): TypedRandomForestClassifier[Inputs] = copy(rf.setSubsamplingRate(value))\n def setFeatureSubsetStrategy(value: FeatureSubsetStrategy): TypedRandomForestClassifier[Inputs] =\n copy(rf.setFeatureSubsetStrategy(value.sparkValue))\n def setMaxBins(value: Int): TypedRandomForestClassifier[Inputs] = copy(rf.setMaxBins(value))\n\n private def copy(newRf: RandomForestClassifier): TypedRandomForestClassifier[Inputs] =\n new TypedRandomForestClassifier[Inputs](newRf, labelCol, featuresCol)\n}\n\nobject TypedRandomForestClassifier {\n case class Outputs(rawPrediction: Vector, probability: Vector, prediction: Double)\n\n def apply[Inputs](implicit inputsChecker: TreesInputsChecker[Inputs]): TypedRandomForestClassifier[Inputs] = {\n new TypedRandomForestClassifier(new RandomForestClassifier(), inputsChecker.labelCol, inputsChecker.featuresCol)\n }\n}\n\n" }, { "path": "ml/src/main/scala/frameless/ml/clustering/TypedBisectingKMeans.scala", "content": "package frameless\npackage ml\npackage classification\n\nimport frameless.ml.internals.VectorInputsChecker\nimport org.apache.spark.ml.clustering.{BisectingKMeans, BisectingKMeansModel}\n\n/**\n * A bisecting k-means algorithm based on the paper \"A comparison of document clustering techniques\"\n * by Steinbach, Karypis, and Kumar, with modification to fit Spark.\n * The algorithm starts from a single cluster that contains all points.\n * Iteratively it finds divisible clusters on the bottom level and bisects each of them using\n * k-means, until there are `k` leaf clusters in total or no leaf clusters are divisible.\n * The bisecting steps of clusters on the same level are grouped together to increase parallelism.\n * If bisecting all divisible clusters on the bottom level would result more than `k` leaf clusters,\n * larger clusters get higher priority.\n *\n * @see \n * Steinbach, Karypis, and Kumar, A comparison of document clustering techniques,\n * KDD Workshop on Text Mining, 2000.\n */\nclass TypedBisectingKMeans[Inputs] private[ml] (\n bkm: BisectingKMeans,\n featuresCol: String\n) extends TypedEstimator[Inputs,TypedBisectingKMeans.Output, BisectingKMeansModel]{\n val estimator: BisectingKMeans =\n bkm\n .setFeaturesCol(featuresCol)\n .setPredictionCol(AppendTransformer.tempColumnName)\n \n def setK(value: Int): TypedBisectingKMeans[Inputs] = copy(bkm.setK(value))\n \n def setMaxIter(value: Int): TypedBisectingKMeans[Inputs] = copy(bkm.setMaxIter(value))\n\n def setMinDivisibleClusterSize(value: Double): TypedBisectingKMeans[Inputs] =\n copy(bkm.setMinDivisibleClusterSize(value))\n \n def setSeed(value: Long): TypedBisectingKMeans[Inputs] = copy(bkm.setSeed(value))\n\n private def copy(newBkm: BisectingKMeans): TypedBisectingKMeans[Inputs] =\n new TypedBisectingKMeans[Inputs](newBkm, featuresCol)\n}\n\nobject TypedBisectingKMeans {\n case class Output(prediction: Int)\n\n def apply[Inputs]()(implicit inputsChecker: VectorInputsChecker[Inputs]): TypedBisectingKMeans[Inputs] =\n new TypedBisectingKMeans(new BisectingKMeans(), inputsChecker.featuresCol)\n}" }, { "path": "ml/src/main/scala/frameless/ml/clustering/TypedKMeans.scala", "content": "package frameless\npackage ml\npackage classification\n\nimport frameless.ml.internals.VectorInputsChecker\nimport frameless.ml.params.kmeans.KMeansInitMode\nimport org.apache.spark.ml.clustering.{KMeans, KMeansModel}\n\n/**\n * K-means clustering with support for k-means|| initialization proposed by Bahmani et al.\n *\n * @see Bahmani et al., Scalable k-means++.\n */\nclass TypedKMeans[Inputs] private[ml] (\n km: KMeans,\n featuresCol: String\n) extends TypedEstimator[Inputs,TypedKMeans.Output,KMeansModel] {\n val estimator: KMeans =\n km\n .setFeaturesCol(featuresCol)\n .setPredictionCol(AppendTransformer.tempColumnName)\n\n def setK(value: Int): TypedKMeans[Inputs] = copy(km.setK(value))\n\n def setInitMode(value: KMeansInitMode): TypedKMeans[Inputs] = copy(km.setInitMode(value.sparkValue))\n\n def setInitSteps(value: Int): TypedKMeans[Inputs] = copy(km.setInitSteps(value))\n\n def setMaxIter(value: Int): TypedKMeans[Inputs] = copy(km.setMaxIter(value))\n\n def setTol(value: Double): TypedKMeans[Inputs] = copy(km.setTol(value))\n\n def setSeed(value: Long): TypedKMeans[Inputs] = copy(km.setSeed(value))\n\n private def copy(newKmeans: KMeans): TypedKMeans[Inputs] = new TypedKMeans[Inputs](newKmeans, featuresCol)\n\n}\n\nobject TypedKMeans{\n case class Output(prediction: Int)\n\n def apply[Inputs](implicit inputsChecker: VectorInputsChecker[Inputs]): TypedKMeans[Inputs] = {\n new TypedKMeans(new KMeans(), inputsChecker.featuresCol)\n }\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/feature/TypedIndexToString.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport frameless.ml.internals.UnaryInputsChecker\nimport org.apache.spark.ml.feature.IndexToString\n\n/**\n * A `TypedTransformer` that maps a column of indices back to a new column of corresponding\n * string values.\n * The index-string mapping must be supplied when creating the `TypedIndexToString`.\n *\n * @see `TypedStringIndexer` for converting strings into indices\n */\nfinal class TypedIndexToString[Inputs] private[ml](indexToString: IndexToString, inputCol: String)\n extends AppendTransformer[Inputs, TypedIndexToString.Outputs, IndexToString] {\n\n val transformer: IndexToString =\n indexToString\n .setInputCol(inputCol)\n .setOutputCol(AppendTransformer.tempColumnName)\n\n}\n\nobject TypedIndexToString {\n case class Outputs(originalOutput: String)\n\n def apply[Inputs](labels: Array[String])\n (implicit inputsChecker: UnaryInputsChecker[Inputs, Double]): TypedIndexToString[Inputs] = {\n new TypedIndexToString[Inputs](new IndexToString().setLabels(labels), inputsChecker.inputCol)\n }\n}" }, { "path": "ml/src/main/scala/frameless/ml/feature/TypedStringIndexer.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport frameless.ml.feature.TypedStringIndexer.HandleInvalid\nimport frameless.ml.internals.UnaryInputsChecker\nimport org.apache.spark.ml.feature.{StringIndexer, StringIndexerModel}\n\n/**\n * A label indexer that maps a string column of labels to an ML column of label indices.\n * The indices are in [0, numLabels), ordered by label frequencies.\n * So the most frequent label gets index 0.\n *\n * @see `TypedIndexToString` for the inverse transformation\n */\nfinal class TypedStringIndexer[Inputs] private[ml](stringIndexer: StringIndexer, inputCol: String)\n extends TypedEstimator[Inputs, TypedStringIndexer.Outputs, StringIndexerModel] {\n\n val estimator: StringIndexer = stringIndexer\n .setInputCol(inputCol)\n .setOutputCol(AppendTransformer.tempColumnName)\n\n def setHandleInvalid(value: HandleInvalid): TypedStringIndexer[Inputs] = copy(stringIndexer.setHandleInvalid(value.sparkValue))\n\n private def copy(newStringIndexer: StringIndexer): TypedStringIndexer[Inputs] =\n new TypedStringIndexer[Inputs](newStringIndexer, inputCol)\n}\n\nobject TypedStringIndexer {\n case class Outputs(indexedOutput: Double)\n\n sealed abstract class HandleInvalid(val sparkValue: String)\n object HandleInvalid {\n case object Error extends HandleInvalid(\"error\")\n case object Skip extends HandleInvalid(\"skip\")\n case object Keep extends HandleInvalid(\"keep\")\n }\n\n def apply[Inputs](implicit inputsChecker: UnaryInputsChecker[Inputs, String]): TypedStringIndexer[Inputs] = {\n new TypedStringIndexer[Inputs](new StringIndexer(), inputsChecker.inputCol)\n }\n}" }, { "path": "ml/src/main/scala/frameless/ml/feature/TypedVectorAssembler.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport org.apache.spark.ml.feature.VectorAssembler\nimport org.apache.spark.ml.linalg.Vector\nimport shapeless.{HList, HNil, LabelledGeneric}\nimport shapeless.ops.hlist.ToTraversable\nimport shapeless.ops.record.{Keys, Values}\nimport shapeless._\nimport scala.annotation.implicitNotFound\n\n/**\n * A feature transformer that merges multiple columns into a vector column.\n */\nfinal class TypedVectorAssembler[Inputs] private[ml](vectorAssembler: VectorAssembler, inputCols: Array[String])\n extends AppendTransformer[Inputs, TypedVectorAssembler.Output, VectorAssembler] {\n\n val transformer: VectorAssembler = vectorAssembler\n .setInputCols(inputCols)\n .setOutputCol(AppendTransformer.tempColumnName)\n\n}\n\nobject TypedVectorAssembler {\n case class Output(vector: Vector)\n\n def apply[Inputs](implicit inputsChecker: TypedVectorAssemblerInputsChecker[Inputs]): TypedVectorAssembler[Inputs] = {\n new TypedVectorAssembler(new VectorAssembler(), inputsChecker.inputCols.toArray)\n }\n}\n\n@implicitNotFound(\n msg = \"Cannot prove that ${Inputs} is a valid input type. Input type must only contain fields of numeric or boolean types.\"\n)\nprivate[ml] trait TypedVectorAssemblerInputsChecker[Inputs] {\n val inputCols: Seq[String]\n}\n\nprivate[ml] object TypedVectorAssemblerInputsChecker {\n implicit def checkInputs[Inputs, InputsRec <: HList, InputsKeys <: HList, InputsVals <: HList](\n implicit\n inputsGen: LabelledGeneric.Aux[Inputs, InputsRec],\n inputsKeys: Keys.Aux[InputsRec, InputsKeys],\n inputsKeysTraverse: ToTraversable.Aux[InputsKeys, Seq, Symbol],\n inputsValues: Values.Aux[InputsRec, InputsVals],\n inputsTypeCheck: TypedVectorAssemblerInputsValueChecker[InputsVals]\n ): TypedVectorAssemblerInputsChecker[Inputs] = new TypedVectorAssemblerInputsChecker[Inputs] {\n val inputCols: Seq[String] = inputsKeys.apply().to[Seq].map(_.name)\n }\n}\n\nprivate[ml] trait TypedVectorAssemblerInputsValueChecker[InputsVals]\n\nprivate[ml] object TypedVectorAssemblerInputsValueChecker {\n implicit def hnilCheckInputsValue: TypedVectorAssemblerInputsValueChecker[HNil] =\n new TypedVectorAssemblerInputsValueChecker[HNil] {}\n\n implicit def hlistCheckInputsValueNumeric[H, T <: HList](\n implicit ch: CatalystNumeric[H],\n tt: TypedVectorAssemblerInputsValueChecker[T]\n ): TypedVectorAssemblerInputsValueChecker[H :: T] = new TypedVectorAssemblerInputsValueChecker[H :: T] {}\n\n implicit def hlistCheckInputsValueBoolean[T <: HList](\n implicit tt: TypedVectorAssemblerInputsValueChecker[T]\n ): TypedVectorAssemblerInputsValueChecker[Boolean :: T] = new TypedVectorAssemblerInputsValueChecker[Boolean :: T] {}\n}\n\n\n" }, { "path": "ml/src/main/scala/frameless/ml/internals/LinearInputsChecker.scala", "content": "package frameless\npackage ml\npackage internals\n\nimport org.apache.spark.ml.linalg._\nimport shapeless.ops.hlist.Length\nimport shapeless.{HList, LabelledGeneric, Nat, Witness}\n\nimport scala.annotation.implicitNotFound\n\n/**\n * Can be used for linear reg algorithm\n */\n@implicitNotFound(\n msg = \"Cannot prove that ${Inputs} is a valid input type. \" +\n \"Input type must only contain a field of type Double (the label) and a field of type \" +\n \"org.apache.spark.ml.linalg.Vector (the features) and optional field of float type (weight).\"\n)\ntrait LinearInputsChecker[Inputs] {\n val featuresCol: String\n val labelCol: String\n val weightCol: Option[String]\n}\n\nobject LinearInputsChecker {\n\n implicit def checkLinearInputs[\n Inputs,\n InputsRec <: HList,\n LabelK <: Symbol,\n FeaturesK <: Symbol](\n implicit\n i0: LabelledGeneric.Aux[Inputs, InputsRec],\n i1: Length.Aux[InputsRec, Nat._2],\n i2: SelectorByValue.Aux[InputsRec, Double, LabelK],\n i3: Witness.Aux[LabelK],\n i4: SelectorByValue.Aux[InputsRec, Vector, FeaturesK],\n i5: Witness.Aux[FeaturesK]\n ): LinearInputsChecker[Inputs] = {\n new LinearInputsChecker[Inputs] {\n val labelCol: String = implicitly[Witness.Aux[LabelK]].value.name\n val featuresCol: String = implicitly[Witness.Aux[FeaturesK]].value.name\n val weightCol: Option[String] = None\n }\n }\n\n implicit def checkLinearInputs2[\n Inputs,\n InputsRec <: HList,\n LabelK <: Symbol,\n FeaturesK <: Symbol,\n WeightK <: Symbol](\n implicit\n i0: LabelledGeneric.Aux[Inputs, InputsRec],\n i1: Length.Aux[InputsRec, Nat._3],\n i2: SelectorByValue.Aux[InputsRec, Vector, FeaturesK],\n i3: Witness.Aux[FeaturesK],\n i4: SelectorByValue.Aux[InputsRec, Double, LabelK],\n i5: Witness.Aux[LabelK],\n i6: SelectorByValue.Aux[InputsRec, Float, WeightK],\n i7: Witness.Aux[WeightK]\n ): LinearInputsChecker[Inputs] = {\n new LinearInputsChecker[Inputs] {\n val labelCol: String = implicitly[Witness.Aux[LabelK]].value.name\n val featuresCol: String = implicitly[Witness.Aux[FeaturesK]].value.name\n val weightCol: Option[String] = Some(implicitly[Witness.Aux[WeightK]].value.name)\n }\n }\n\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/internals/SelectorByValue.scala", "content": "package frameless\npackage ml\npackage internals\n\nimport shapeless.labelled.FieldType\nimport shapeless.{::, DepFn1, HList, Witness}\n\n/**\n * Typeclass supporting record selection by value type (returning the first key whose value is of type `Value`)\n */\ntrait SelectorByValue[L <: HList, Value] extends DepFn1[L] with Serializable { type Out <: Symbol }\n\nobject SelectorByValue {\n type Aux[L <: HList, Value, Out0 <: Symbol] = SelectorByValue[L, Value] { type Out = Out0 }\n\n implicit def select[K <: Symbol, T <: HList, Value](implicit wk: Witness.Aux[K]): Aux[FieldType[K, Value] :: T, Value, K] = {\n new SelectorByValue[FieldType[K, Value] :: T, Value] {\n type Out = K\n def apply(l: FieldType[K, Value] :: T): Out = wk.value\n }\n }\n\n implicit def recurse[H, T <: HList, Value](implicit st: SelectorByValue[T, Value]): Aux[H :: T, Value, st.Out] = {\n new SelectorByValue[H :: T, Value] {\n type Out = st.Out\n def apply(l: H :: T): Out = st(l.tail)\n }\n }\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/internals/TreesInputsChecker.scala", "content": "package frameless\npackage ml\npackage internals\n\nimport shapeless.ops.hlist.Length\nimport shapeless.{HList, LabelledGeneric, Nat, Witness}\nimport org.apache.spark.ml.linalg._\n\nimport scala.annotation.implicitNotFound\n\n/**\n * Can be used for all tree-based ML algorithm (decision tree, random forest, gradient-boosted trees)\n */\n@implicitNotFound(\n msg = \"Cannot prove that ${Inputs} is a valid input type. \" +\n \"Input type must only contain a field of type Double (the label) and a field of type \" +\n \"org.apache.spark.ml.linalg.Vector (the features).\"\n)\ntrait TreesInputsChecker[Inputs] {\n val featuresCol: String\n val labelCol: String\n}\n\nobject TreesInputsChecker {\n\n implicit def checkTreesInputs[\n Inputs,\n InputsRec <: HList,\n LabelK <: Symbol,\n FeaturesK <: Symbol](\n implicit\n i0: LabelledGeneric.Aux[Inputs, InputsRec],\n i1: Length.Aux[InputsRec, Nat._2],\n i2: SelectorByValue.Aux[InputsRec, Double, LabelK],\n i3: Witness.Aux[LabelK],\n i4: SelectorByValue.Aux[InputsRec, Vector, FeaturesK],\n i5: Witness.Aux[FeaturesK]\n ): TreesInputsChecker[Inputs] = {\n new TreesInputsChecker[Inputs] {\n val labelCol: String = implicitly[Witness.Aux[LabelK]].value.name\n val featuresCol: String = implicitly[Witness.Aux[FeaturesK]].value.name\n }\n }\n\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/internals/UnaryInputsChecker.scala", "content": "package frameless\npackage ml\npackage internals\n\nimport shapeless.ops.hlist.Length\nimport shapeless.{HList, LabelledGeneric, Nat, Witness}\n\nimport scala.annotation.implicitNotFound\n\n/**\n * Can be used for all unary transformers (i.e almost all of them)\n */\n@implicitNotFound(\n msg = \"Cannot prove that ${Inputs} is a valid input type. Input type must have only one field of type ${Expected}\"\n)\ntrait UnaryInputsChecker[Inputs, Expected] {\n val inputCol: String\n}\n\nobject UnaryInputsChecker {\n\n implicit def checkUnaryInputs[Inputs, Expected, InputsRec <: HList, InputK <: Symbol](\n implicit\n i0: LabelledGeneric.Aux[Inputs, InputsRec],\n i1: Length.Aux[InputsRec, Nat._1],\n i2: SelectorByValue.Aux[InputsRec, Expected, InputK],\n i3: Witness.Aux[InputK]\n ): UnaryInputsChecker[Inputs, Expected] = new UnaryInputsChecker[Inputs, Expected] {\n val inputCol: String = implicitly[Witness.Aux[InputK]].value.name\n }\n\n}\n\n" }, { "path": "ml/src/main/scala/frameless/ml/internals/VectorInputsChecker.scala", "content": "package frameless\npackage ml\npackage internals\n\nimport shapeless.ops.hlist.Length\nimport shapeless.{HList, LabelledGeneric, Nat, Witness}\n\nimport scala.annotation.implicitNotFound\nimport org.apache.spark.ml.linalg.Vector\n\n/** Can be used whenever algorithm requires only vector */\n@implicitNotFound(\n msg = \"Cannot prove that ${Inputs} is a valid input type. \" +\n \"Input type must only contain a field of type org.apache.spark.ml.linalg.Vector (the features).\"\n)\ntrait VectorInputsChecker[Inputs] {\n val featuresCol: String\n}\n\nobject VectorInputsChecker {\n implicit def checkVectorInput[Inputs, InputsRec <: HList, FeaturesK <: Symbol](\n implicit\n i0: LabelledGeneric.Aux[Inputs, InputsRec],\n i1: Length.Aux[InputsRec, Nat._1],\n i2: SelectorByValue.Aux[InputsRec, Vector, FeaturesK],\n i3: Witness.Aux[FeaturesK]\n ): VectorInputsChecker[Inputs] = {\n new VectorInputsChecker[Inputs] {\n val featuresCol: String = i3.value.name\n }\n }\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/package.scala", "content": "package frameless\n\nimport org.apache.spark.sql.FramelessInternals.UserDefinedType\nimport org.apache.spark.ml.FramelessInternals\nimport org.apache.spark.ml.linalg.{Matrix, Vector}\n\npackage object ml {\n\n implicit val mlVectorUdt: UserDefinedType[Vector] = FramelessInternals.vectorUdt\n\n implicit val mlMatrixUdt: UserDefinedType[Matrix] = FramelessInternals.matrixUdt\n\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/params/kmeans/KMeansInitMode.scala", "content": "package frameless\npackage ml\npackage params\npackage kmeans\n\n/**\n * Param for the initialization algorithm.\n * This can be either \"random\" to choose random points as\n * initial cluster centers, or \"k-means||\" to use a parallel variant of k-means++\n * (Bahmani et al., Scalable K-Means++, VLDB 2012).\n * Default: k-means||.\n */\n\nsealed abstract class KMeansInitMode private[ml](val sparkValue: String)\n\nobject KMeansInitMode {\n case object Random extends KMeansInitMode(\"random\")\n case object KMeansPlusPlus extends KMeansInitMode(\"k-means||\")\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/params/linears/LossStrategy.scala", "content": "package frameless\npackage ml\npackage params\npackage linears\n/**\n * SquaredError measures the average of the squares of the errors—that is,\n * the average squared difference between the estimated values and what is estimated.\n *\n * Huber Loss loss function less sensitive to outliers in data than the\n * squared error loss\n */\nsealed abstract class LossStrategy private[ml](val sparkValue: String)\nobject LossStrategy {\n case object SquaredError extends LossStrategy(\"squaredError\")\n case object Huber extends LossStrategy(\"huber\")\n}\n" }, { "path": "ml/src/main/scala/frameless/ml/params/linears/Solver.scala", "content": "package frameless\npackage ml\npackage params\npackage linears\n\n/**\n * solver algorithm used for optimization.\n * - \"l-bfgs\" denotes Limited-memory BFGS which is a limited-memory quasi-Newton\n * optimization method.\n * - \"normal\" denotes using Normal Equation as an analytical solution to the linear regression\n * problem. This solver is limited to `LinearRegression.MAX_FEATURES_FOR_NORMAL_SOLVER`.\n * - \"auto\" (default) means that the solver algorithm is selected automatically.\n * The Normal Equations solver will be used when possible, but this will automatically fall\n * back to iterative optimization methods when needed.\n *\n * spark\n */\n\nsealed abstract class Solver private[ml](val sparkValue: String)\nobject Solver {\n case object LBFGS extends Solver(\"l-bfgs\")\n case object Auto extends Solver(\"auto\")\n case object Normal extends Solver(\"normal\")\n}\n\n" }, { "path": "ml/src/main/scala/frameless/ml/params/trees/FeatureSubsetStrategy.scala", "content": "package frameless\npackage ml\npackage params\npackage trees\n/**\n * The number of features to consider for splits at each tree node.\n * Supported options:\n * - Auto: Choose automatically for task:\n * If numTrees == 1, set to All\n * If numTrees > 1 (forest), set to Sqrt for classification and\n * to OneThird for regression.\n * - All: use all features\n * - OneThird: use 1/3 of the features\n * - Sqrt: use sqrt(number of features)\n * - Log2: use log2(number of features)\n * - Ratio: use (ratio * number of features) features\n * - NumberOfFeatures: use numberOfFeatures features.\n * (default = Auto)\n *\n * These various settings are based on the following references:\n * - log2: tested in Breiman (2001)\n * - sqrt: recommended by Breiman manual for random forests\n * - The defaults of sqrt (classification) and onethird (regression) match the R randomForest\n * package.\n *\n * @see Breiman (2001)\n * @see \n * Breiman manual for random forests\n */\nsealed abstract class FeatureSubsetStrategy private[ml](val sparkValue: String)\nobject FeatureSubsetStrategy {\n case object Auto extends FeatureSubsetStrategy(\"auto\")\n case object All extends FeatureSubsetStrategy(\"all\")\n case object OneThird extends FeatureSubsetStrategy(\"onethird\")\n case object Sqrt extends FeatureSubsetStrategy(\"sqrt\")\n case object Log2 extends FeatureSubsetStrategy(\"log2\")\n case class Ratio(value: Double) extends FeatureSubsetStrategy(value.toString)\n case class NumberOfFeatures(value: Int) extends FeatureSubsetStrategy(value.toString)\n}" }, { "path": "ml/src/main/scala/frameless/ml/regression/TypedLinearRegression.scala", "content": "package frameless\npackage ml\npackage regression\n\nimport frameless.ml.internals.LinearInputsChecker\nimport frameless.ml.params.linears.{LossStrategy, Solver}\nimport frameless.ml.{AppendTransformer, TypedEstimator}\nimport org.apache.spark.ml.regression.{LinearRegression, LinearRegressionModel}\n\n/**\n * Linear Regression linear approach to modelling the relationship\n * between a scalar response (or dependent variable) and one or more explanatory variables\n */\nfinal class TypedLinearRegression [Inputs] private[ml](\n lr: LinearRegression,\n labelCol: String,\n featuresCol: String,\n weightCol: Option[String]\n) extends TypedEstimator[Inputs, TypedLinearRegression.Outputs, LinearRegressionModel] {\n\n val estimatorWithoutWeight : LinearRegression = lr\n .setLabelCol(labelCol)\n .setFeaturesCol(featuresCol)\n .setPredictionCol(AppendTransformer.tempColumnName)\n\n val estimator = if (weightCol.isDefined) estimatorWithoutWeight.setWeightCol(weightCol.get) else estimatorWithoutWeight\n\n def setRegParam(value: Double): TypedLinearRegression[Inputs] = copy(lr.setRegParam(value))\n def setFitIntercept(value: Boolean): TypedLinearRegression[Inputs] = copy(lr.setFitIntercept(value))\n def setStandardization(value: Boolean): TypedLinearRegression[Inputs] = copy(lr.setStandardization(value))\n def setElasticNetParam(value: Double): TypedLinearRegression[Inputs] = copy(lr.setElasticNetParam(value))\n def setMaxIter(value: Int): TypedLinearRegression[Inputs] = copy(lr.setMaxIter(value))\n def setTol(value: Double): TypedLinearRegression[Inputs] = copy(lr.setTol(value))\n def setSolver(value: Solver): TypedLinearRegression[Inputs] = copy(lr.setSolver(value.sparkValue))\n def setAggregationDepth(value: Int): TypedLinearRegression[Inputs] = copy(lr.setAggregationDepth(value))\n def setLoss(value: LossStrategy): TypedLinearRegression[Inputs] = copy(lr.setLoss(value.sparkValue))\n def setEpsilon(value: Double): TypedLinearRegression[Inputs] = copy(lr.setEpsilon(value))\n\n private def copy(newLr: LinearRegression): TypedLinearRegression[Inputs] =\n new TypedLinearRegression[Inputs](newLr, labelCol, featuresCol, weightCol)\n\n}\n\nobject TypedLinearRegression {\n case class Outputs(prediction: Double)\n case class Weight(weight: Double)\n\n\n def apply[Inputs](implicit inputsChecker: LinearInputsChecker[Inputs]): TypedLinearRegression[Inputs] = {\n new TypedLinearRegression(new LinearRegression(), inputsChecker.labelCol, inputsChecker.featuresCol, inputsChecker.weightCol)\n }\n}" }, { "path": "ml/src/main/scala/frameless/ml/regression/TypedRandomForestRegressor.scala", "content": "package frameless\npackage ml\npackage regression\n\nimport frameless.ml.internals.TreesInputsChecker\nimport frameless.ml.params.trees.FeatureSubsetStrategy\nimport org.apache.spark.ml.regression.{RandomForestRegressionModel, RandomForestRegressor}\n\n/**\n * Random Forest\n * learning algorithm for regression.\n * It supports both continuous and categorical features.\n */\nfinal class TypedRandomForestRegressor[Inputs] private[ml](\n rf: RandomForestRegressor,\n labelCol: String,\n featuresCol: String\n) extends TypedEstimator[Inputs, TypedRandomForestRegressor.Outputs, RandomForestRegressionModel] {\n\n val estimator: RandomForestRegressor =\n rf\n .setLabelCol(labelCol)\n .setFeaturesCol(featuresCol)\n .setPredictionCol(AppendTransformer.tempColumnName)\n\n def setNumTrees(value: Int): TypedRandomForestRegressor[Inputs] = copy(rf.setNumTrees(value))\n def setMaxDepth(value: Int): TypedRandomForestRegressor[Inputs] = copy(rf.setMaxDepth(value))\n def setMinInfoGain(value: Double): TypedRandomForestRegressor[Inputs] = copy(rf.setMinInfoGain(value))\n def setMinInstancesPerNode(value: Int): TypedRandomForestRegressor[Inputs] = copy(rf.setMinInstancesPerNode(value))\n def setMaxMemoryInMB(value: Int): TypedRandomForestRegressor[Inputs] = copy(rf.setMaxMemoryInMB(value))\n def setSubsamplingRate(value: Double): TypedRandomForestRegressor[Inputs] = copy(rf.setSubsamplingRate(value))\n def setFeatureSubsetStrategy(value: FeatureSubsetStrategy): TypedRandomForestRegressor[Inputs] =\n copy(rf.setFeatureSubsetStrategy(value.sparkValue))\n def setMaxBins(value: Int): TypedRandomForestRegressor[Inputs] = copy(rf.setMaxBins(value))\n\n private def copy(newRf: RandomForestRegressor): TypedRandomForestRegressor[Inputs] =\n new TypedRandomForestRegressor[Inputs](newRf, labelCol, featuresCol)\n}\n\nobject TypedRandomForestRegressor {\n case class Outputs(prediction: Double)\n\n def apply[Inputs](implicit inputsChecker: TreesInputsChecker[Inputs])\n : TypedRandomForestRegressor[Inputs] = {\n new TypedRandomForestRegressor(new RandomForestRegressor(), inputsChecker.labelCol, inputsChecker.featuresCol)\n }\n}" }, { "path": "ml/src/main/scala/org/apache/spark/ml/FramelessInternals.scala", "content": "package org.apache.spark.ml\n\nimport org.apache.spark.ml.linalg.{MatrixUDT, VectorUDT}\n\nobject FramelessInternals {\n\n // because org.apache.spark.ml.linalg.VectorUDT is private[spark]\n val vectorUdt = new VectorUDT\n\n // because org.apache.spark.ml.linalg.MatrixUDT is private[spark]\n val matrixUdt = new MatrixUDT\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/FramelessMlSuite.scala", "content": "package frameless\npackage ml\n\nimport org.scalactic.anyvals.PosZInt\nimport org.scalatest.BeforeAndAfterAll\nimport org.scalatestplus.scalacheck.Checkers\nimport org.scalatest.funsuite.AnyFunSuite\n\nclass FramelessMlSuite extends AnyFunSuite with Checkers with BeforeAndAfterAll with SparkTesting {\n // Limit size of generated collections and number of checks because Travis\n implicit override val generatorDrivenConfig =\n PropertyCheckConfiguration(sizeRange = PosZInt(10), minSize = PosZInt(10))\n implicit val sparkDelay: SparkDelay[Job] = Job.framelessSparkDelayForJob\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/Generators.scala", "content": "package frameless\npackage ml\n\nimport frameless.ml.params.linears.{LossStrategy, Solver}\nimport frameless.ml.params.trees.FeatureSubsetStrategy\nimport org.apache.spark.ml.linalg.{Matrices, Matrix, Vector, Vectors}\nimport org.scalacheck.{Arbitrary, Gen}\n\nobject Generators {\n\n implicit val arbVector: Arbitrary[Vector] = Arbitrary {\n val genDenseVector = Gen.listOf(arbDouble.arbitrary).suchThat(_.nonEmpty).map(doubles => Vectors.dense(doubles.toArray))\n val genSparseVector = genDenseVector.map(_.toSparse)\n\n Gen.oneOf(genDenseVector, genSparseVector)\n }\n\n implicit val arbMatrix: Arbitrary[Matrix] = Arbitrary {\n Gen.sized { size =>\n for {\n nbRows <- Gen.choose(0, size)\n nbCols <- Gen.choose(1, size)\n matrix <- {\n Gen.listOfN(nbRows * nbCols, arbDouble.arbitrary)\n .map(values => Matrices.dense(nbRows, nbCols, values.toArray))\n }\n } yield matrix\n }\n }\n\n implicit val arbTreesFeaturesSubsetStrategy: Arbitrary[FeatureSubsetStrategy] = Arbitrary {\n val genRatio = Gen.choose(0D, 1D).suchThat(_ > 0D).map(FeatureSubsetStrategy.Ratio)\n val genNumberOfFeatures = Gen.choose(1, Int.MaxValue).map(FeatureSubsetStrategy.NumberOfFeatures)\n\n Gen.oneOf(Gen.const(FeatureSubsetStrategy.All),\n Gen.const(FeatureSubsetStrategy.All),\n Gen.const(FeatureSubsetStrategy.Log2),\n Gen.const(FeatureSubsetStrategy.OneThird),\n Gen.const(FeatureSubsetStrategy.Sqrt),\n genRatio,\n genNumberOfFeatures\n )\n }\n\n implicit val arbLossStrategy: Arbitrary[LossStrategy] = Arbitrary {\n Gen.const(LossStrategy.SquaredError)\n }\n\n implicit val arbSolver: Arbitrary[Solver] = Arbitrary {\n Gen.oneOf(\n Gen.const(Solver.LBFGS),\n Gen.const(Solver.Auto),\n Gen.const(Solver.Normal)\n )\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/TypedEncoderInstancesTests.scala", "content": "package frameless\npackage ml\n\nimport org.scalacheck.Prop._\nimport org.apache.spark.ml.linalg._\nimport org.apache.spark.ml.regression.DecisionTreeRegressor\nimport Generators._\nimport scala.util.Random\n\nclass TypedEncoderInstancesTests extends FramelessMlSuite {\n\n test(\"Vector encoding is injective using collect()\") {\n val prop = forAll { vector: Vector =>\n TypedDataset.create(Seq(vector)).collect().run() == Seq(vector)\n }\n check(prop)\n }\n\n test(\"Matrix encoding is injective using collect()\") {\n val prop = forAll { matrix: Matrix =>\n TypedDataset.create(Seq(matrix)).collect().run() == Seq(matrix)\n }\n check(prop)\n }\n\n test(\"Vector is encoded as VectorUDT and thus can be run in a Spark ML model\") {\n case class Input(features: Vector, label: Double)\n\n val prop = forAll { trainingData: Matrix =>\n (trainingData.numRows >= 1) ==> {\n val inputs = trainingData.rowIter.toVector.map(vector => Input(vector, 0D))\n val inputsDS = TypedDataset.create(inputs)\n\n val model = new DecisionTreeRegressor()\n\n // this line would throw a runtime exception if Vector was not encoded as VectorUDT\n val trainedModel = model.fit(inputsDS.dataset)\n\n val randomInput = inputs(Random.nextInt(inputs.length))\n val randomInputDS = TypedDataset.create(Seq(randomInput))\n\n val prediction = trainedModel.transform(randomInputDS.dataset)\n .select(\"prediction\")\n .head()\n .getAs[Double](0)\n\n prediction == 0D\n }\n\n }\n\n check(prop, MinSize(1))\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/classification/ClassificationIntegrationTests.scala", "content": "package frameless\npackage ml\npackage classification\n\nimport frameless.ml.feature.{TypedIndexToString, TypedStringIndexer, TypedVectorAssembler}\nimport org.apache.spark.ml.linalg.Vector\nimport org.scalatest.matchers.must.Matchers\n\nclass ClassificationIntegrationTests extends FramelessMlSuite with Matchers {\n\n test(\"predict field3 from field1 and field2 using a RandomForestClassifier\") {\n case class Data(field1: Double, field2: Int, field3: String)\n\n // Training\n\n val trainingDataDs = TypedDataset.create(Seq.fill(10)(Data(0D, 10, \"foo\")))\n\n case class Features(field1: Double, field2: Int)\n val vectorAssembler = TypedVectorAssembler[Features]\n\n case class DataWithFeatures(field1: Double, field2: Int, field3: String, features: Vector)\n val dataWithFeatures = vectorAssembler.transform(trainingDataDs).as[DataWithFeatures]()\n\n case class StringIndexerInput(field3: String)\n val indexer = TypedStringIndexer[StringIndexerInput]\n val indexerModel = indexer.fit(dataWithFeatures).run()\n\n case class IndexedDataWithFeatures(field1: Double, field2: Int, field3: String, features: Vector, indexedField3: Double)\n val indexedData = indexerModel.transform(dataWithFeatures).as[IndexedDataWithFeatures]()\n\n case class RFInputs(indexedField3: Double, features: Vector)\n val rf = TypedRandomForestClassifier[RFInputs]\n\n val model = rf.fit(indexedData).run()\n\n // Prediction\n\n val testData = TypedDataset.create(Seq(\n Data(0D, 10, \"foo\")\n ))\n val testDataWithFeatures = vectorAssembler.transform(testData).as[DataWithFeatures]()\n val indexedTestData = indexerModel.transform(testDataWithFeatures).as[IndexedDataWithFeatures]()\n\n case class PredictionInputs(features: Vector, indexedField3: Double)\n val testInput = indexedTestData.project[PredictionInputs]\n\n case class PredictionResultIndexed(\n features: Vector,\n indexedField3: Double,\n rawPrediction: Vector,\n probability: Vector,\n predictedField3Indexed: Double\n )\n val predictionDs = model.transform(testInput).as[PredictionResultIndexed]()\n\n case class IndexToStringInput(predictedField3Indexed: Double)\n val indexToString = TypedIndexToString[IndexToStringInput](indexerModel.transformer.labelsArray.flatten)\n\n case class PredictionResult(\n features: Vector,\n indexedField3: Double,\n rawPrediction: Vector,\n probability: Vector,\n predictedField3Indexed: Double,\n predictedField3: String\n )\n val stringPredictionDs = indexToString.transform(predictionDs).as[PredictionResult]()\n\n val prediction = stringPredictionDs.select(stringPredictionDs.col('predictedField3)).collect().run().toList\n\n prediction mustEqual List(\"foo\")\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/classification/TypedRandomForestClassifierTests.scala", "content": "package frameless\npackage ml\npackage classification\n\nimport shapeless.test.illTyped\nimport org.apache.spark.ml.linalg._\nimport frameless.ml.params.trees.FeatureSubsetStrategy\nimport org.scalacheck.{Arbitrary, Gen}\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.must.Matchers\n\nclass TypedRandomForestClassifierTests extends FramelessMlSuite with Matchers {\n implicit val arbDouble: Arbitrary[Double] =\n Arbitrary(Gen.choose(1, 99).map(_.toDouble)) // num classes must be between 0 and 100 for the test\n implicit val arbVectorNonEmpty: Arbitrary[Vector] =\n Arbitrary(Generators.arbVector.arbitrary suchThat (_.size > 0)) // vector must not be empty for RandomForestClassifier\n import Generators.arbTreesFeaturesSubsetStrategy\n\n test(\"fit() returns a correct TypedTransformer\") {\n val prop = forAll { x2: X2[Double, Vector] =>\n val rf = TypedRandomForestClassifier[X2[Double, Vector]]\n val ds = TypedDataset.create(Seq(x2))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X5[Double, Vector, Vector, Vector, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n\n val prop2 = forAll { x2: X2[Vector, Double] =>\n val rf = TypedRandomForestClassifier[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x2))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X5[Vector, Double, Vector, Vector, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n\n def prop3[A: TypedEncoder: Arbitrary] = forAll { x3: X3[Vector, Double, A] =>\n val rf = TypedRandomForestClassifier[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x3))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X6[Vector, Double, A, Vector, Vector, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b), pDs.col('c)).collect().run() == Seq((x3.a, x3.b, x3.c))\n }\n\n check(prop)\n check(prop2)\n check(prop3[String])\n check(prop3[Double])\n }\n\n test(\"param setting is retained\") {\n val prop = forAll { featureSubsetStrategy: FeatureSubsetStrategy =>\n val rf = TypedRandomForestClassifier[X2[Double, Vector]]\n .setNumTrees(10)\n .setMaxBins(100)\n .setFeatureSubsetStrategy(featureSubsetStrategy)\n .setMaxDepth(10)\n .setMaxMemoryInMB(100)\n .setMinInfoGain(0.1D)\n .setMinInstancesPerNode(2)\n .setSubsamplingRate(0.9D)\n\n val ds = TypedDataset.create(Seq(X2(0D, Vectors.dense(0D))))\n val model = rf.fit(ds).run()\n\n model.transformer.getNumTrees == 10 &&\n model.transformer.getMaxBins == 100 &&\n model.transformer.getFeatureSubsetStrategy == featureSubsetStrategy.sparkValue &&\n model.transformer.getMaxDepth == 10 &&\n model.transformer.getMaxMemoryInMB == 100 &&\n model.transformer.getMinInfoGain == 0.1D &&\n model.transformer.getMinInstancesPerNode == 2 &&\n model.transformer.getSubsamplingRate == 0.9D\n }\n\n check(prop)\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedRandomForestClassifier.create[Double]()\")\n illTyped(\"TypedRandomForestClassifier.create[X1[Double]]()\")\n illTyped(\"TypedRandomForestClassifier.create[X2[Double, Double]]()\")\n illTyped(\"TypedRandomForestClassifier.create[X3[Vector, Double, Int]]()\")\n illTyped(\"TypedRandomForestClassifier.create[X2[Vector, String]]()\")\n }\n\n}" }, { "path": "ml/src/test/scala/frameless/ml/clustering/BisectingKMeansTests.scala", "content": "package frameless\npackage ml\npackage clustering\n\nimport frameless.{TypedDataset, TypedEncoder, X1, X2, X3}\nimport frameless.ml.classification.TypedBisectingKMeans\nimport org.scalacheck.Arbitrary\nimport org.apache.spark.ml.linalg._\nimport org.scalacheck.Prop._\nimport frameless.ml._\nimport org.scalatest.matchers.must.Matchers\n\nclass BisectingKMeansTests extends FramelessMlSuite with Matchers {\n implicit val arbVector: Arbitrary[Vector] =\n Arbitrary(Generators.arbVector.arbitrary)\n\n test(\"fit() returns a correct TypedTransformer\") {\n val prop = forAll { x1: X1[Vector] =>\n val km = TypedBisectingKMeans[X1[Vector]]()\n val ds = TypedDataset.create(Seq(x1))\n val model = km.fit(ds).run()\n val pDs = model.transform(ds).as[X2[Vector, Int]]()\n\n pDs.select(pDs.col('a)).collect().run().toList == Seq(x1.a)\n }\n\n def prop3[A: TypedEncoder : Arbitrary] = forAll { x2: X2[Vector, A] =>\n val km = TypedBisectingKMeans[X1[Vector]]()\n val ds = TypedDataset.create(Seq(x2))\n val model = km.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Vector, A, Int]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq((x2.a, x2.b))\n }\n\n check(prop)\n check(prop3[Double])\n }\n\n test(\"param setting is retained\") {\n val rf = TypedBisectingKMeans[X1[Vector]]()\n .setK(10)\n .setMaxIter(10)\n .setMinDivisibleClusterSize(1)\n .setSeed(123332)\n\n val ds = TypedDataset.create(Seq(X2(Vectors.dense(Array(0D)),0)))\n val model = rf.fit(ds).run()\n\n model.transformer.getK == 10 &&\n model.transformer.getMaxIter == 10 &&\n model.transformer.getMinDivisibleClusterSize == 1 &&\n model.transformer.getSeed == 123332\n }\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/clustering/ClusteringIntegrationTests.scala", "content": "package frameless\npackage ml\npackage clustering\n\nimport frameless.ml.FramelessMlSuite\nimport frameless.ml.classification.{TypedBisectingKMeans, TypedKMeans}\nimport org.apache.spark.ml.linalg.Vector\nimport frameless._\nimport frameless.ml._\nimport frameless.ml.feature._\nimport org.scalatest.matchers.must.Matchers\n\nclass ClusteringIntegrationTests extends FramelessMlSuite with Matchers {\n\n test(\"predict field2 from field1 using a K-means clustering\") {\n // Training\n val trainingDataDs = TypedDataset.create(Seq.fill(5)(X2(10D, 0)) :+ X2(100D,0))\n\n val vectorAssembler = TypedVectorAssembler[X1[Double]]\n\n val dataWithFeatures = vectorAssembler.transform(trainingDataDs).as[X3[Double,Int,Vector]]()\n\n case class Input(c: Vector)\n val km = TypedKMeans[Input].setK(2)\n\n val model = km.fit(dataWithFeatures).run()\n\n // Prediction\n val testSeq = Seq(\n X2(10D, 0),\n X2(100D, 1)\n )\n\n val testData = TypedDataset.create(testSeq)\n val testDataWithFeatures = vectorAssembler.transform(testData).as[X3[Double,Int,Vector]]()\n\n val predictionDs = model.transform(testDataWithFeatures).as[X4[Double,Int,Vector,Int]]()\n\n val prediction = predictionDs.select(predictionDs.col[Int]('d)).collect().run().toList\n\n prediction mustEqual testSeq.map(_.b)\n }\n\n test(\"predict field2 from field1 using a bisecting K-means clustering\") {\n // Training\n val trainingDataDs = TypedDataset.create(Seq.fill(5)(X2(10D, 0)) :+ X2(100D,0))\n\n val vectorAssembler = TypedVectorAssembler[X1[Double]]\n\n val dataWithFeatures = vectorAssembler.transform(trainingDataDs).as[X3[Double, Int, Vector]]()\n\n case class Inputs(c: Vector)\n val bkm = TypedBisectingKMeans[Inputs]().setK(2)\n\n val model = bkm.fit(dataWithFeatures).run()\n\n // Prediction\n val testSeq = Seq(\n X2(10D, 0),\n X2(100D, 1)\n )\n\n val testData = TypedDataset.create(testSeq)\n val testDataWithFeatures = vectorAssembler.transform(testData).as[X3[Double, Int, Vector]]()\n\n val predictionDs = model.transform(testDataWithFeatures).as[X4[Double,Int,Vector,Int]]()\n\n val prediction = predictionDs.select(predictionDs.col[Int]('d)).collect().run().toList\n\n prediction mustEqual testSeq.map(_.b)\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/clustering/KMeansTests.scala", "content": "package frameless\npackage ml\npackage clustering\n\nimport frameless.ml.classification.TypedKMeans\nimport frameless.{TypedDataset, TypedEncoder, X1, X2, X3}\nimport org.apache.spark.ml.linalg._\nimport org.scalacheck.{Arbitrary, Gen}\nimport org.scalacheck.Prop._\nimport frameless.ml._\nimport frameless.ml.params.kmeans.KMeansInitMode\nimport org.scalatest.matchers.must.Matchers\n\nclass KMeansTests extends FramelessMlSuite with Matchers {\n implicit val arbVector: Arbitrary[Vector] =\n Arbitrary(Generators.arbVector.arbitrary)\n implicit val arbKMeansInitMode: Arbitrary[KMeansInitMode] =\n Arbitrary {\n Gen.oneOf(\n Gen.const(KMeansInitMode.KMeansPlusPlus),\n Gen.const(KMeansInitMode.Random)\n )\n }\n\n /**\n * copies a vector as we need two rows of the right dimension for 3.4's alg\n */\n def newRowWithSameDimension(vect: Vector): Vector = {\n val dubs = vect.toArray.map(_ % 2) // k is two\n val dense = Vectors.dense(dubs)\n vect match {\n case _: SparseVector => dense.toSparse\n case _ => dense\n }\n }\n\n test(\"fit() returns a correct TypedTransformer\") {\n val prop = forAll { x1: X1[Vector] =>\n val x1a = X1(newRowWithSameDimension(x1.a))\n val km = TypedKMeans[X1[Vector]]\n val ds = TypedDataset.create(Seq(x1, x1a))\n\n val model = km.fit(ds).run()\n val pDs = model.transform(ds).as[X2[Vector, Int]]()\n\n pDs.select(pDs.col('a)).collect().run().toList == Seq(x1.a, x1a.a)\n }\n\n def prop3[A: TypedEncoder : Arbitrary] = forAll { x2: X2[Vector, A] =>\n val x2a = x2.copy(a = newRowWithSameDimension(x2.a))\n val km = TypedKMeans[X1[Vector]]\n val ds = TypedDataset.create(Seq(x2, x2a))\n val model = km.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Vector, A, Int]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run().toList == Seq((x2.a, x2.b), (x2a.a, x2a.b))\n }\n\n tolerantRun( _.isInstanceOf[ArrayIndexOutOfBoundsException] ) {\n check(prop)\n check(prop3[Double])\n }\n }\n\n test(\"param setting is retained\") {\n val prop = forAll { initMode: KMeansInitMode =>\n val rf = TypedKMeans[X1[Vector]]\n .setInitMode(KMeansInitMode.Random)\n .setInitSteps(2)\n .setK(10)\n .setMaxIter(15)\n .setSeed(123223L)\n .setTol(12D)\n\n val ds = TypedDataset.create(Seq(X2(Vectors.dense(Array(0D)), 0)))\n val model = rf.fit(ds).run()\n\n model.transformer.getInitMode == KMeansInitMode.Random.sparkValue &&\n model.transformer.getInitSteps == 2 &&\n model.transformer.getK == 10 &&\n model.transformer.getMaxIter == 15 &&\n model.transformer.getSeed == 123223L &&\n model.transformer.getTol == 12D\n }\n\n check(prop)\n }\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/feature/TypedIndexToStringTests.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport org.scalacheck.{Arbitrary, Gen}\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\nimport org.scalatest.matchers.must.Matchers\n\nclass TypedIndexToStringTests extends FramelessMlSuite with Matchers {\n\n test(\".transform() correctly transform an input dataset\") {\n implicit val arbDouble = Arbitrary(Gen.choose(0, 99).map(_.toDouble))\n\n def prop[A: TypedEncoder: Arbitrary] = forAll { x2: X2[Double, A] =>\n val transformer = TypedIndexToString[X1[Double]](Array.fill(100)(\"foo\"))\n val ds = TypedDataset.create(Seq(x2))\n val ds2 = transformer.transform(ds)\n\n ds2.collect().run() == Seq((x2.a, x2.b, \"foo\"))\n }\n\n check(prop[Double])\n check(prop[String])\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedIndexToString.create[String](Array(\\\"foo\\\"))\")\n illTyped(\"TypedIndexToString.create[X1[String]](Array(\\\"foo\\\"))\")\n illTyped(\"TypedIndexToString.create[X1[Long]](Array(\\\"foo\\\"))\")\n illTyped(\"TypedIndexToString.create[X2[String, Int]](Array(\\\"foo\\\"))\")\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/feature/TypedStringIndexerTests.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport frameless.ml.feature.TypedStringIndexer.HandleInvalid\nimport org.scalacheck.{Arbitrary, Gen}\nimport org.scalacheck.Prop._\nimport shapeless.test.illTyped\nimport org.scalatest.matchers.must.Matchers\n\nclass TypedStringIndexerTests extends FramelessMlSuite with Matchers {\n\n test(\".fit() returns a correct TypedTransformer\") {\n def prop[A: TypedEncoder : Arbitrary] = forAll { x2: X2[String, A] =>\n val indexer = TypedStringIndexer[X1[String]]\n val ds = TypedDataset.create(Seq(x2))\n val model = indexer.fit(ds).run()\n val resultDs = model.transform(ds).as[X3[String, A, Double]]()\n\n resultDs.collect().run() == Seq(X3(x2.a, x2.b, 0D))\n }\n\n check(prop[Double])\n check(prop[String])\n }\n\n test(\"param setting is retained\") {\n implicit val arbHandleInvalid: Arbitrary[HandleInvalid] = Arbitrary {\n Gen.oneOf(HandleInvalid.Keep, HandleInvalid.Error, HandleInvalid.Skip)\n }\n\n val prop = forAll { handleInvalid: HandleInvalid =>\n val indexer = TypedStringIndexer[X1[String]]\n .setHandleInvalid(handleInvalid)\n val ds = TypedDataset.create(Seq(X1(\"foo\")))\n val model = indexer.fit(ds).run()\n\n model.transformer.getHandleInvalid == handleInvalid.sparkValue\n }\n\n check(prop)\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedStringIndexer.create[Double]()\")\n illTyped(\"TypedStringIndexer.create[X1[Double]]()\")\n illTyped(\"TypedStringIndexer.create[X2[String, Long]]()\")\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/feature/TypedVectorAssemblerTests.scala", "content": "package frameless\npackage ml\npackage feature\n\nimport org.scalacheck.Arbitrary\nimport org.scalacheck.Prop._\nimport org.apache.spark.ml.linalg._\nimport shapeless.test.illTyped\n\nclass TypedVectorAssemblerTests extends FramelessMlSuite {\n\n test(\".transform() returns a correct TypedTransformer\") {\n def prop[A: TypedEncoder: Arbitrary] = forAll { x5: X5[Int, Long, Double, Boolean, A] =>\n val assembler = TypedVectorAssembler[X4[Int, Long, Double, Boolean]]\n val ds = TypedDataset.create(Seq(x5))\n val ds2 = assembler.transform(ds).as[X6[Int, Long, Double, Boolean, A, Vector]]()\n\n ds2.collect().run() ==\n Seq(X6(x5.a, x5.b, x5.c, x5.d, x5.e, Vectors.dense(x5.a.toDouble, x5.b.toDouble, x5.c, if (x5.d) 1D else 0D)))\n }\n\n def prop2[A: TypedEncoder: Arbitrary] = forAll { x5: X5[Boolean, BigDecimal, Byte, Short, A] =>\n val assembler = TypedVectorAssembler[X4[Boolean, BigDecimal, Byte, Short]]\n val ds = TypedDataset.create(Seq(x5))\n val ds2 = assembler.transform(ds).as[X6[Boolean, BigDecimal, Byte, Short, A, Vector]]()\n\n ds2.collect().run() ==\n Seq(X6(x5.a, x5.b, x5.c, x5.d, x5.e, Vectors.dense(if (x5.a) 1D else 0D, x5.b.toDouble, x5.c.toDouble, x5.d.toDouble)))\n }\n\n check(prop[String])\n check(prop[Double])\n check(prop2[Long])\n check(prop2[Boolean])\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedVectorAssembler.create[Double]()\")\n illTyped(\"TypedVectorAssembler.create[X1[String]]()\")\n illTyped(\"TypedVectorAssembler.create[X2[String, Double]]()\")\n illTyped(\"TypedVectorAssembler.create[X3[Int, String, Double]]()\")\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/regression/RegressionIntegrationTests.scala", "content": "package frameless\npackage ml\npackage regression\n\nimport frameless.ml.feature.TypedVectorAssembler\nimport org.apache.spark.ml.linalg.Vector\nimport org.scalatest.matchers.must.Matchers\n\nclass RegressionIntegrationTests extends FramelessMlSuite with Matchers {\n\n test(\"predict field3 from field1 and field2 using a RandomForestRegressor\") {\n case class Data(field1: Double, field2: Int, field3: Double)\n\n // Training\n\n val trainingDataDs = TypedDataset.create(Seq.fill(10)(Data(0D, 10, 0D)))\n\n case class Features(field1: Double, field2: Int)\n val vectorAssembler = TypedVectorAssembler[Features]\n\n case class DataWithFeatures(field1: Double, field2: Int, field3: Double, features: Vector)\n val dataWithFeatures = vectorAssembler.transform(trainingDataDs).as[DataWithFeatures]()\n\n case class RFInputs(field3: Double, features: Vector)\n val rf = TypedRandomForestRegressor[RFInputs]\n\n val model = rf.fit(dataWithFeatures).run()\n\n // Prediction\n\n val testData = TypedDataset.create(Seq(\n Data(0D, 10, 0D)\n ))\n val testDataWithFeatures = vectorAssembler.transform(testData).as[DataWithFeatures]()\n\n case class PredictionResult(field1: Double, field2: Int, field3: Double, features: Vector, predictedField3: Double)\n val predictionDs = model.transform(testDataWithFeatures).as[PredictionResult]()\n\n val prediction = predictionDs.select(predictionDs.col('predictedField3)).collect().run().toList\n\n prediction mustEqual List(0D)\n }\n\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/regression/TypedLinearRegressionTests.scala", "content": "package frameless\npackage ml\npackage regression\n\nimport frameless.ml.params.linears.{LossStrategy, Solver}\nimport org.apache.spark.ml.linalg._\nimport org.scalacheck.Arbitrary\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.should.Matchers\nimport shapeless.test.illTyped\n\nclass TypedLinearRegressionTests extends FramelessMlSuite with Matchers {\n\n implicit val arbVectorNonEmpty: Arbitrary[Vector] = Arbitrary(Generators.arbVector.arbitrary)\n\n test(\"fit() returns a correct TypedTransformer\") {\n val prop = forAll { x2: X2[Double, Vector] =>\n val lr = TypedLinearRegression[X2[Double, Vector]]\n val ds = TypedDataset.create(Seq(x2))\n\n val model = lr.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Double, Vector, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n val prop2 = forAll { x2: X2[Vector, Double] =>\n val lr = TypedLinearRegression[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x2))\n val model = lr.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Vector, Double, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n\n def prop3[A: TypedEncoder: Arbitrary] = forAll { x3: X3[Vector, Double, A] =>\n val lr = TypedLinearRegression[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x3))\n val model = lr.fit(ds).run()\n val pDs = model.transform(ds).as[X4[Vector, Double, A, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b), pDs.col('c)).collect().run() == Seq((x3.a, x3.b, x3.c))\n }\n\n check(prop)\n check(prop2)\n check(prop3[String])\n check(prop3[Double])\n }\n\n test(\"param setting is retained\") {\n import Generators.{arbLossStrategy, arbSolver}\n\n val prop = forAll { (lossStrategy: LossStrategy, solver: Solver) =>\n val lr = TypedLinearRegression[X2[Double, Vector]]\n .setAggregationDepth(10)\n .setEpsilon(4)\n .setFitIntercept(true)\n .setLoss(lossStrategy)\n .setMaxIter(23)\n .setRegParam(1.2)\n .setStandardization(true)\n .setTol(2.3)\n .setSolver(solver)\n\n val ds = TypedDataset.create(Seq(X2(0D, Vectors.dense(0D))))\n val model = lr.fit(ds).run()\n\n model.transformer.getAggregationDepth == 10 &&\n model.transformer.getEpsilon == 4.0 &&\n model.transformer.getLoss == lossStrategy.sparkValue &&\n model.transformer.getMaxIter == 23 &&\n model.transformer.getRegParam == 1.2 &&\n model.transformer.getTol == 2.3 &&\n model.transformer.getSolver == solver.sparkValue\n }\n\n check(prop)\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedLinearRegressor.create[Double]()\")\n illTyped(\"TypedLinearRegressor.create[X1[Double]]()\")\n illTyped(\"TypedLinearRegressor.create[X2[Double, Double]]()\")\n illTyped(\"TypedLinearRegressor.create[X3[Vector, Double, Int]]()\")\n illTyped(\"TypedLinearRegressor.create[X2[Vector, String]]()\")\n }\n\n test(\"TypedLinearRegressor should fit straight line \") {\n case class Point(features: Vector, labels: Double)\n\n val ds = Seq(\n X2(new DenseVector(Array(1.0)): Vector, 1.0),\n X2(new DenseVector(Array(2.0)): Vector, 2.0),\n X2(new DenseVector(Array(3.0)): Vector, 3.0),\n X2(new DenseVector(Array(4.0)): Vector, 4.0),\n X2(new DenseVector(Array(5.0)): Vector, 5.0),\n X2(new DenseVector(Array(6.0)): Vector, 6.0)\n )\n\n val ds2 = Seq(\n X3(new DenseVector(Array(1.0)): Vector,2F, 1.0),\n X3(new DenseVector(Array(2.0)): Vector,2F, 2.0),\n X3(new DenseVector(Array(3.0)): Vector,2F, 3.0),\n X3(new DenseVector(Array(4.0)): Vector,2F, 4.0),\n X3(new DenseVector(Array(5.0)): Vector,2F, 5.0),\n X3(new DenseVector(Array(6.0)): Vector,2F, 6.0)\n )\n\n val tds = TypedDataset.create(ds)\n\n val lr = TypedLinearRegression[X2[Vector, Double]]\n .setMaxIter(10)\n\n val model = lr.fit(tds).run()\n\n val tds2 = TypedDataset.create(ds2)\n\n val lr2 = TypedLinearRegression[X3[Vector, Float, Double]]\n .setMaxIter(10)\n\n val model2 = lr2.fit(tds2).run()\n\n model.transformer.coefficients shouldEqual new DenseVector(Array(1.0))\n model2.transformer.coefficients shouldEqual new DenseVector(Array(1.0))\n }\n}\n" }, { "path": "ml/src/test/scala/frameless/ml/regression/TypedRandomForestRegressorTests.scala", "content": "package frameless\npackage ml\npackage regression\n\nimport frameless.ml.params.trees.FeatureSubsetStrategy\nimport shapeless.test.illTyped\nimport org.apache.spark.ml.linalg._\nimport org.scalacheck.Arbitrary\nimport org.scalacheck.Prop._\nimport org.scalatest.matchers.must.Matchers\n\nclass TypedRandomForestRegressorTests extends FramelessMlSuite with Matchers {\n implicit val arbVectorNonEmpty: Arbitrary[Vector] =\n Arbitrary(Generators.arbVector.arbitrary suchThat (_.size > 0)) // vector must not be empty for RandomForestRegressor\n import Generators.arbTreesFeaturesSubsetStrategy\n\n test(\"fit() returns a correct TypedTransformer\") {\n val prop = forAll { x2: X2[Double, Vector] =>\n val rf = TypedRandomForestRegressor[X2[Double, Vector]]\n val ds = TypedDataset.create(Seq(x2))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Double, Vector, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n\n val prop2 = forAll { x2: X2[Vector, Double] =>\n val rf = TypedRandomForestRegressor[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x2))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X3[Vector, Double, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b)).collect().run() == Seq(x2.a -> x2.b)\n }\n\n def prop3[A: TypedEncoder: Arbitrary] = forAll { x3: X3[Vector, Double, A] =>\n val rf = TypedRandomForestRegressor[X2[Vector, Double]]\n val ds = TypedDataset.create(Seq(x3))\n val model = rf.fit(ds).run()\n val pDs = model.transform(ds).as[X4[Vector, Double, A, Double]]()\n\n pDs.select(pDs.col('a), pDs.col('b), pDs.col('c)).collect().run() == Seq((x3.a, x3.b, x3.c))\n }\n\n check(prop)\n check(prop2)\n check(prop3[String])\n check(prop3[Double])\n }\n\n test(\"param setting is retained\") {\n val prop = forAll { featureSubsetStrategy: FeatureSubsetStrategy =>\n val rf = TypedRandomForestRegressor[X2[Double, Vector]]\n .setNumTrees(10)\n .setMaxBins(100)\n .setFeatureSubsetStrategy(featureSubsetStrategy)\n .setMaxDepth(10)\n .setMaxMemoryInMB(100)\n .setMinInfoGain(0.1D)\n .setMinInstancesPerNode(2)\n .setSubsamplingRate(0.9D)\n\n val ds = TypedDataset.create(Seq(X2(0D, Vectors.dense(0D))))\n val model = rf.fit(ds).run()\n\n model.transformer.getNumTrees == 10 &&\n model.transformer.getMaxBins == 100 &&\n model.transformer.getFeatureSubsetStrategy == featureSubsetStrategy.sparkValue &&\n model.transformer.getMaxDepth == 10 &&\n model.transformer.getMaxMemoryInMB == 100 &&\n model.transformer.getMinInfoGain == 0.1D &&\n model.transformer.getMinInstancesPerNode == 2 &&\n model.transformer.getSubsamplingRate == 0.9D\n }\n\n check(prop)\n }\n\n test(\"create() compiles only with correct inputs\") {\n illTyped(\"TypedRandomForestRegressor.create[Double]()\")\n illTyped(\"TypedRandomForestRegressor.create[X1[Double]]()\")\n illTyped(\"TypedRandomForestRegressor.create[X2[Double, Double]]()\")\n illTyped(\"TypedRandomForestRegressor.create[X3[Vector, Double, Int]]()\")\n illTyped(\"TypedRandomForestRegressor.create[X2[Vector, String]]()\")\n }\n\n}\n" }, { "path": "project/Common.scala", "content": "import sbt.Keys._\nimport sbt._\nimport sbt.plugins.JvmPlugin\n\nimport org.scalafmt.sbt.ScalafmtPlugin.autoImport._\n\nobject Common extends AutoPlugin {\n override def trigger = allRequirements\n override def requires = JvmPlugin\n\n override def projectSettings = Seq(\n scalafmtFilter := \"diff-ref=78f708d\"\n )\n}\n" }, { "path": "project/build.properties", "content": "sbt.version=1.12.11\n" }, { "path": "project/plugins.sbt", "content": "val sbtTypelevelVersion = \"0.8.5\"\n\naddSbtPlugin(\"org.typelevel\" % \"sbt-typelevel-ci-release\" % sbtTypelevelVersion)\n\naddSbtPlugin(\"org.typelevel\" % \"sbt-typelevel-site\" % sbtTypelevelVersion)\n\naddSbtPlugin(\"org.scoverage\" % \"sbt-scoverage\" % \"2.4.4\")\n\naddSbtPlugin(\"org.scalameta\" % \"sbt-scalafmt\" % \"2.5.6\")\n\naddSbtPlugin(\"com.timushev.sbt\" % \"sbt-updates\" % \"0.6.4\")\n" }, { "path": "refined/src/main/scala/frameless/refined/RefinedFieldEncoders.scala", "content": "package frameless.refined\n\nimport scala.reflect.ClassTag\n\nimport org.apache.spark.sql.catalyst.expressions._\nimport org.apache.spark.sql.catalyst.expressions.objects.{\n Invoke, NewInstance, UnwrapOption, WrapOption\n}\nimport org.apache.spark.sql.types._\n\nimport eu.timepit.refined.api.RefType\n\nimport frameless.{ TypedEncoder, RecordFieldEncoder }\n\nprivate[refined] trait RefinedFieldEncoders {\n /**\n * @tparam T the refined type (e.g. `String`)\n */\n implicit def optionRefined[F[_, _], T, R](\n implicit\n i0: RefType[F],\n i1: TypedEncoder[T],\n i2: ClassTag[F[T, R]],\n ): RecordFieldEncoder[Option[F[T, R]]] =\n RecordFieldEncoder[Option[F[T, R]]](new TypedEncoder[Option[F[T, R]]] {\n def nullable = true\n\n // `Refined` is a Value class: https://github.com/fthomas/refined/blob/master/modules/core/shared/src/main/scala-3.0-/eu/timepit/refined/api/Refined.scala#L8\n def jvmRepr = ObjectType(classOf[Option[F[T, R]]])\n\n def catalystRepr: DataType = i1.catalystRepr\n\n val innerJvmRepr = ObjectType(i2.runtimeClass)\n\n def fromCatalyst(path: Expression): Expression = {\n val javaValue = i1.fromCatalyst(path)\n val value = NewInstance(i2.runtimeClass, Seq(javaValue), innerJvmRepr)\n\n WrapOption(value, innerJvmRepr)\n }\n\n @inline def toCatalyst(path: Expression): Expression = {\n val value = UnwrapOption(innerJvmRepr, path)\n\n val javaValue = Invoke(value, \"value\", i1.jvmRepr, Nil)\n\n i1.toCatalyst(javaValue)\n }\n\n override def toString = s\"optionRefined[${i2.runtimeClass.getName}]\"\n })\n\n /**\n * @tparam T the refined type (e.g. `String`)\n */\n implicit def refined[F[_, _], T, R](\n implicit\n i0: RefType[F],\n i1: TypedEncoder[T],\n i2: ClassTag[F[T, R]],\n ): RecordFieldEncoder[F[T, R]] =\n RecordFieldEncoder[F[T, R]](new TypedEncoder[F[T, R]] {\n def nullable = i1.nullable\n\n // `Refined` is a Value class: https://github.com/fthomas/refined/blob/master/modules/core/shared/src/main/scala-3.0-/eu/timepit/refined/api/Refined.scala#L8\n def jvmRepr = i1.jvmRepr\n\n def catalystRepr: DataType = i1.catalystRepr\n\n def fromCatalyst(path: Expression): Expression =\n i1.fromCatalyst(path)\n\n @inline def toCatalyst(path: Expression): Expression =\n i1.toCatalyst(path)\n\n override def toString = s\"refined[${i2.runtimeClass.getName}]\"\n })\n}\n\n" }, { "path": "refined/src/main/scala/frameless/refined/package.scala", "content": "package frameless\n\nimport scala.reflect.ClassTag\n\nimport eu.timepit.refined.api.{ RefType, Validate }\n\npackage object refined extends RefinedFieldEncoders {\n implicit def refinedInjection[F[_, _], T, R](\n implicit\n refType: RefType[F],\n validate: Validate[T, R]\n ): Injection[F[T, R], T] = Injection(\n refType.unwrap,\n { value =>\n refType.refine[R](value) match {\n case Left(errMsg) =>\n throw new IllegalArgumentException(\n s\"Value $value does not satisfy refinement predicate: $errMsg\")\n\n case Right(res) => res\n }\n })\n\n implicit def refinedEncoder[F[_, _], T, R](\n implicit\n i0: RefType[F],\n i1: Validate[T, R],\n i2: TypedEncoder[T],\n i3: ClassTag[F[T, R]]\n ): TypedEncoder[F[T, R]] = TypedEncoder.usingInjection(\n i3, refinedInjection, i2)\n}\n\n" }, { "path": "refined/src/test/scala/frameless/RefinedFieldEncoderTests.scala", "content": "package frameless\n\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.types.{\n IntegerType, ObjectType, StringType, StructField, StructType\n}\n\nimport org.scalatest.matchers.should.Matchers\n\nclass RefinedFieldEncoderTests extends TypedDatasetSuite with Matchers {\n test(\"Encode a bare refined type\") {\n import eu.timepit.refined.auto._\n import eu.timepit.refined.types.string.NonEmptyString\n\n val encoder: TypedEncoder[NonEmptyString] = {\n import frameless.refined.refinedEncoder\n TypedEncoder[NonEmptyString]\n }\n\n val ss = session\n import ss.implicits._\n\n encoder.catalystRepr shouldBe StringType\n\n val nes: NonEmptyString = \"Non Empty String\"\n\n val unsafeDs = TypedDataset.createUnsafe(sc.parallelize(Seq(nes.value)).toDF())(encoder)\n\n val expected = Seq(nes)\n\n unsafeDs.collect().run() shouldBe expected\n }\n\n test(\"Encode case class with a refined field\") {\n import RefinedTypesTests._\n\n // Check jvmRepr\n import org.apache.spark.sql.types.ObjectType\n\n encoderA.jvmRepr shouldBe ObjectType(classOf[A])\n\n // Check catalystRepr\n val expectedAStructType = StructType(Seq(\n StructField(\"a\", IntegerType, false),\n StructField(\"s\", StringType, false)))\n\n encoderA.catalystRepr shouldBe expectedAStructType\n\n // Check unsafe\n val unsafeDs: TypedDataset[A] = {\n val rdd = sc.parallelize(Seq(Row(as.a, as.s.toString)))\n val df = session.createDataFrame(rdd, expectedAStructType)\n\n TypedDataset.createUnsafe(df)(encoderA)\n }\n\n val expected = Seq(as)\n\n unsafeDs.collect().run() shouldBe expected\n\n // Check safe\n val safeDs = TypedDataset.create(expected)\n\n safeDs.collect().run() shouldBe expected\n }\n\n test(\"Encode case class with a refined optional field\") {\n import RefinedTypesTests._\n\n // Check jvmRepr\n encoderB.jvmRepr shouldBe ObjectType(classOf[B])\n\n // Check catalystRepr\n val expectedBStructType = StructType(Seq(\n StructField(\"a\", IntegerType, false),\n StructField(\"s\", StringType, true)))\n\n encoderB.catalystRepr shouldBe expectedBStructType\n\n // Check unsafe\n val unsafeDs: TypedDataset[B] = {\n val rdd = sc.parallelize(Seq(\n Row(bs.a, bs.s.mkString),\n Row(2, null.asInstanceOf[String]),\n ))\n\n val df = session.createDataFrame(rdd, expectedBStructType)\n\n TypedDataset.createUnsafe(df)(encoderB)\n }\n\n val expected = Seq(bs, B(2, None))\n\n unsafeDs.collect().run() shouldBe expected\n\n // Check safe\n val safeDs = TypedDataset.create(expected)\n\n safeDs.collect().run() shouldBe expected\n }\n}\n\nobject RefinedTypesTests {\n import eu.timepit.refined.auto._\n import eu.timepit.refined.types.string.NonEmptyString\n\n case class A(a: Int, s: NonEmptyString)\n case class B(a: Int, s: Option[NonEmptyString])\n\n val nes: NonEmptyString = \"Non Empty String\"\n\n val as = A(-42, nes)\n val bs = B(-42, Option(nes))\n\n import frameless.refined._ // implicit instances for refined\n\n implicit val encoderA: TypedEncoder[A] = TypedEncoder.usingDerivation\n\n implicit val encoderB: TypedEncoder[B] = TypedEncoder.usingDerivation\n}\n" }, { "path": "scripts/docs-build.sh", "content": "#!/bin/bash\n\nset -eux\n\nsbt copyReadme mdoc\n\ngitbook=\"node_modules/gitbook-cli/bin/gitbook.js\"\n\nif ! test -e $gitbook; then\n npm install gitbook\n npm install gitbook-cli\nfi\n\n$gitbook build mdocs/target/mdoc docs/book\n\nmv docs/book/* .\n\nexit 0\n" }, { "path": "scripts/docs-publish.sh", "content": "#!/bin/bash\n\nset -eux\n\n# Check that the working directory is a git repository and the repository has no outstanding changes.\ngit diff-index --quiet HEAD\n\ncommit=$(git show -s --format=%h)\n\ngit checkout gh-pages\n\ngit merge \"$commit\"\n\nbash scripts/docs-build.sh\n\ngit add .\n\ngit commit -am \"Rebuild documentation ($commit)\"\n\necho \"Verify that you didn't break anything:\"\necho \" $ python -m SimpleHTTPServer 8000\"\necho \" $ xdg-open http://localhost:8000/\"\necho \"\"\necho \"Then push to the gh-pages branch:\"\necho \" $ git push gh-pages\"\n" }, { "path": "scripts/travis-publish.sh", "content": "#!/bin/bash\n\n# Taken + modified from typelevel/cats\n# https://github.com/typelevel/cats/blob/a8a7587f558541cbabc5c40053181928b4baf78c/scripts/travis-publish.sh\n\nexport publish_cmd=\"publishLocal\"\n\n# if [[ $TRAVIS_PULL_REQUEST == \"false\" && $TRAVIS_BRANCH == \"master\" && $(cat version.sbt) =~ \"-SNAPSHOT\" ]]; then\n# export publish_cmd=\"common/publish cats/publish dataset/publish dataframe/publish\"\n# fi\n\nsbt_cmd=\"sbt ++$TRAVIS_SCALA_VERSION -Dfile.encoding=UTF8 -J-XX:ReservedCodeCacheSize=256M\"\n\ncase \"$PHASE\" in\n A) \n docs_cmd=\"$sbt_cmd doc tut\"\n run_cmd=\"$docs_cmd\"\n ;;\n B)\n coverage=\"$sbt_cmd coverage test && sbt coverageReport && bash <(curl -s https://codecov.io/bash)\"\n run_cmd=\"$coverage\"\n ;;\n C) \n run_cmd=\"$sbt_cmd clean $publish_cmd\"\n ;; \nesac \neval $run_cmd\n" } ]